Script exit with exit code 2
fatal error: concurrent map writes
goroutine 52 [running]:
runtime.throw(0xab7b31, 0x15)
/usr/local/go/src/runtime/panic.go:1116 +0x72 fp=0xc000e3ba90 sp=0xc000e3ba60 pc=0x453732
runtime.mapassign_faststr(0xa13de0, 0xc000235410, 0xc00007f7e3, 0x18, 0xaa04f9)
/usr/local/go/src/runtime/map_faststr.go:211 +0x3f7 fp=0xc000e3baf8 sp=0xc000e3ba90 pc=0x4338e7
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f7a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:314 +0x5a9 fp=0xc000e3bfb8 sp=0xc000e3baf8 pc=0x824ef9
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1373 +0x1 fp=0xc000e3bfc0 sp=0xc000e3bfb8 pc=0x483571
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1 [semacquire]:
os.lstatNolog(0xc0010dc000, 0x5b, 0x5b, 0x1, 0xc0010dc000, 0x5b)
/usr/local/go/src/os/stat_unix.go:41 +0x31
os.Lstat(0xc0010dc000, 0x5b, 0x2, 0xc0010dc000, 0x5b, 0x0)
/usr/local/go/src/os/stat.go:22 +0x4d
path/filepath.walk(0xc002eea730, 0x42, 0xb89740, 0xc000bd9930, 0xc00018d548, 0x0, 0x0)
/usr/local/go/src/path/filepath/path.go:378 +0x1d3
path/filepath.walk(0xc000117880, 0x3f, 0xb89740, 0xc000236b60, 0xc00018d548, 0x0, 0x0)
/usr/local/go/src/path/filepath/path.go:384 +0x2ff
path/filepath.walk(0x7ffdeec2c7c9, 0x37, 0xb89740, 0xc000236a90, 0xc00018d548, 0x0, 0xc00018d248)
/usr/local/go/src/path/filepath/path.go:384 +0x2ff
path/filepath.Walk(0x7ffdeec2c7c9, 0x37, 0xc00018d548, 0x10, 0x0)
/usr/local/go/src/path/filepath/path.go:406 +0xff
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand(0x7ffdeec2c7c9, 0x37, 0xffffffffffffffff, 0xbfb68fb6b5d50d55)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:265 +0x1a1
main.main()
/home/ec2-user/environment/cluster2/tgnews.go:75 +0x72a
goroutine 6 [syscall]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:147 +0x9c
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.Notify.func1
/usr/local/go/src/os/signal/signal.go:127 +0x44
goroutine 8 [chan receive]:
main.main.func1(0xc000073140, 0xbfb68fb6b5d50d55, 0x396bbd, 0x1251a20)
/home/ec2-user/environment/cluster2/tgnews.go:36 +0x41
created by main.main
/home/ec2-user/environment/cluster2/tgnews.go:35 +0x176
goroutine 9 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000b7d400, 0xaa04f9, 0x2, 0xc003f10f80, 0x1a06, 0xc000b7d240, 0x42d308, 0x8)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords.func1(0x0, 0xc002c12510)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:174 +0x86
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc002c124e0, 0xc000b7d2c8, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords(0xc000b7d388)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:173 +0xb0
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc000b7d388, 0xc000cca6c0, 0xaa04f9, 0x2, 0x42d308, 0x8, 0x9ecc80, 0xc000b7d301, 0xc0039560f8)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:43 +0xf5
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc000b7d518, 0xc000cca6c0, 0xaa04f9, 0x2, 0x0, 0x0, 0xc00009c4c8, 0x44c51f, 0x7fe238b70130)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007e420, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 10 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc001b2d600, 0x857, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc001b2d600, 0x857, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc001b2d600, 0x857, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc001b2d600, 0x857, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc001b2d600, 0x857, 0x0, 0x0, 0x857, 0xc001b2d600, 0x857)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc001b2d600, 0x857, 0x857, 0xa80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000c52090, 0x2b, 0xc001b2d600, 0x857, 0xc001b2d600, 0x857, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000c52090, 0x2b, 0xc001b2d600, 0x857, 0xaa0378, 0x1, 0xc001b2d600, 0x857, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007e4e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 11 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc001c1e000, 0xf8a, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:87 +0x379
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc001c1e000, 0xf8a, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc001c1e000, 0xf8a, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc001c1e000, 0xf8a, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc001c1e000, 0xf8a, 0x0, 0x0, 0xf8a, 0xc001c1e000, 0xf8a)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc001c1e000, 0xf8a, 0xf8a, 0x1300)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000e78ee0, 0x61, 0xc001c1e000, 0xf8a, 0xc001c1e000, 0xf8a, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000e78ee0, 0x61, 0xc001c1e000, 0xf8a, 0xaa0378, 0x1, 0xc001c1e000, 0xf8a, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007e5a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 12 [semacquire]:
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc000c80ff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x67e
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc000c80ff8, 0x1, 0xc002068010, 0xc0102c4aa01, 0x280460fec7d0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc000c80ff8, 0x4, 0x188, 0xc0039e5e01, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc0039e5000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc000237040, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc002830360, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc000c818f8, 0xc0004aa000, 0xe71, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007e660, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 13 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc000236dd0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc000236ea0, 0xc003999b20, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00229d580, 0x0, 0x0, 0x0, 0xc000a75200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00229d580, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc000a758f8, 0xc000945000, 0x17b1, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007e720, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 14 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc000ad6a80, 0x97c, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:81 +0x208
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc000ad6a80, 0x97c, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc000ad6a80, 0x97c, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc000ad6a80, 0x97c, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc000ad6a80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc000ad6a80, 0x97c, 0x0, 0x0, 0x97c, 0xc00, 0xc000ad6a80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc000ad6a80, 0x97c, 0x97c, 0xc00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000ccd0c0, 0x3b, 0xc000ad6a80, 0x97c, 0xc000ad6a80, 0x97c, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000ccd0c0, 0x3b, 0xc000ad6a80, 0x97c, 0xaa0378, 0x1, 0xc000ad6a80, 0x97c, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007e7e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 15 [semacquire]:
bytes.makeSlice(0x31a, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc002f066f0, 0x8, 0x5)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc002f066f0, 0xc00150c280, 0x8, 0x6, 0xffffffffffffffff, 0xaa042b)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc002f066f0, 0xc00150c280, 0x8, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc002f066f0, 0xc0035b2c40, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:169 +0x3a3
golang.org/x/net/html.render1(0xb84d20, 0xc002f066f0, 0xc0035b27e0, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc002f066f0, 0xc0035b2770, 0xb84d20, 0xc002f066f0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc002f066f0, 0xc0035b2770, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0042e7a10, 0x50d201, 0xc0010a83f0, 0x87, 0x3f)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000ac5518, 0xc00137fe80, 0xc005bcec40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:285 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007e8a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 16 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe144005a80, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0021df2c0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0021df2a0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002843c00, 0x65a, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0008a0000, 0x2d, 0xc002843c00, 0x65a, 0xc002843c00, 0x65a, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0008a0000, 0x2d, 0xc002843c00, 0x65a, 0xaa0378, 0x1, 0xc002843c00, 0x65a, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007e960, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 18 [semacquire]:
bytes.makeSlice(0xd0, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc000adf1b0, 0xd0, 0xc000adf0d0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc000adf1b0, 0xc000ce4340, 0xd0, 0x4, 0x4, 0xc000adf1d8)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00101ef50)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00101eee0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc003dae660, 0x1, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords.func1(0x0, 0xc003dae660)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:174 +0x4a
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc003dae630, 0xc000adf2c8, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords(0xc000adf388)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:173 +0xb0
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc000adf388, 0xc0041ebf80, 0xaa04f9, 0x2, 0x42d308, 0x8, 0x9ecc80, 0xc000adf301, 0xc0015c3310)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:43 +0xf5
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc000adf518, 0xc0041ebf80, 0xaa04f9, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007ea20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 19 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe220004e70, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0001b7270, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0001b7260, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001d1b000, 0x7df, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000c52180, 0x26, 0xc001d1b000, 0x7df, 0xc001d1b000, 0x7df, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000c52180, 0x26, 0xc001d1b000, 0x7df, 0xaa0378, 0x1, 0xc001d1b000, 0x7df, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007eae0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 20 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe128639f60, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0028f15c0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0028f15b0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0010c3900, 0x4bb, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000f63090, 0x48, 0xc0010c3900, 0x4bb, 0xc0010c3900, 0x4bb, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000f63090, 0x48, 0xc0010c3900, 0x4bb, 0xaa0378, 0x1, 0xc0010c3900, 0x4bb, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007eba0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 21 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc00148d600, 0x822, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:81 +0x208
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc00148d600, 0x822, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc00148d600, 0x822, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc00148d600, 0x822, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00148d600, 0x822, 0x0, 0x0, 0x822, 0xc00148d600, 0x822)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00148d600, 0x822, 0x822, 0x900)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00322f140, 0x3d, 0xc00148d600, 0x822, 0xc00148d600, 0x822, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00322f140, 0x3d, 0xc00148d600, 0x822, 0xaa0378, 0x1, 0xc00148d600, 0x822, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007ec60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 25 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc003113a60, 0xc000e5d220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000e5d588, 0xaa04f9, 0x2, 0xc001c43950, 0xe2, 0x1, 0x38, 0xc00103e720)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc000e5d518, 0xc00000e780, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007ef60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 26 [semacquire]:
strings.genSplit(0xc00582db00, 0x8b6, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x1, 0xc000480001, 0x1a3)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*ContentExtractor).isHighLinkDensity(0xc00031f518, 0xc000f5f650, 0x2)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:567 +0xd5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00031f518, 0xc00000e160, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:386 +0x1ae
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f020, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 34 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc000392000, 0xc000390000)
/home/develop/go/src/github.com/patrickmn/go-cache/cache.go:1079 +0xd5
created by github.com/patrickmn/go-cache.runJanitor
/home/develop/go/src/github.com/patrickmn/go-cache/cache.go:1099 +0xa9
goroutine 27 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe234009c10, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0012a9680, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0012a9670, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00127a000, 0x7e2, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000a9e2c0, 0x3c, 0xc00127a000, 0x7e2, 0xc00127a000, 0x7e2, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000a9e2c0, 0x3c, 0xc00127a000, 0x7e2, 0xaa0378, 0x1, 0xc00127a000, 0x7e2, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f0e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 28 [semacquire]:
bytes.makeSlice(0xe00, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0006cf0e8, 0x200, 0x400)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).ReadFrom(0xc0006cf0e8, 0xb7b620, 0xc0000adad0, 0x203000, 0x203000, 0x203000)
/usr/local/go/src/bytes/buffer.go:202 +0x48
io/ioutil.readAll(0xb7b620, 0xc0000adad0, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:36 +0xe3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc0000adad0, 0xc0006cf1d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0000adad0, 0x0, 0x0, 0x0, 0xc00, 0xc0016f7400, 0xbe6, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc0016f7400, 0xbe6, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc001217cf0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0006cf518, 0xc000c44ae0, 0xc005c54da0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f1a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 30 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe23400cb80, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0042f3ca0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0042f3c90, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00239a000, 0x7ff, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00087bc80, 0x3a, 0xc00239a000, 0x7ff, 0xc00239a000, 0x7ff, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00087bc80, 0x3a, 0xc00239a000, 0x7ff, 0xaa0378, 0x1, 0xc00239a000, 0x7ff, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f320, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 31 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1ac0014c0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002ff6700, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002ff66f0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002fc6000, 0x815, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0012726c0, 0x3e, 0xc002fc6000, 0x815, 0xc002fc6000, 0x815, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0012726c0, 0x3e, 0xc002fc6000, 0x815, 0xaa0378, 0x1, 0xc002fc6000, 0x815, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f3e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 32 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0014f7500, 0x6b7, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0014f7500, 0x6b7, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0014f7500, 0x6b7, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0014f7500, 0x6b7, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc0014f7500)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0014f7500, 0x6b7, 0x0, 0x0, 0x6b7, 0x800, 0xc0014f7500)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0014f7500, 0x6b7, 0x6b7, 0x800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000cccdc0, 0x38, 0xc0014f7500, 0x6b7, 0xc0014f7500, 0x6b7, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000cccdc0, 0x38, 0xc0014f7500, 0x6b7, 0xaa0378, 0x1, 0xc0014f7500, 0x6b7, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f4a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 50 [runnable]:
strings.genSplit(0xc006762000, 0xa5f, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x1300, 0xc003fd3500, 0xa5f)
/usr/local/go/src/strings/strings.go:245 +0x2b0
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0006c3588, 0xaa04f9, 0x2, 0xc003fd3500, 0xa5f, 0x31, 0xe5, 0xc001c5edc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0006c3518, 0xc0009c60c0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f620, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 51 [semacquire]:
bytes.makeSlice(0x5a, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0025ab198, 0x5a, 0x42ca96)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0025ab198, 0xc000479500, 0x5a, 0x28, 0x28, 0xc0051876b0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc001394230)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0013941c0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc00424a210, 0xc0051876b0, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc0025ab518, 0xc00424a210, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:527 +0xc5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0025ab518, 0xc0000b3f80, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f6e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 53 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc005877800, 0x74b, 0x5165d7, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc005877800, 0x74b, 0xc0040969e4, 0x4)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0021233c0, 0xaa04f9, 0x2, 0xc005877800, 0x74b, 0xc005877800, 0xc0021233b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005877800, 0x74b, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002123518, 0xc00039ef60, 0x23, 0xc000e7a630)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f860, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 54 [semacquire]:
regexp.(*Regexp).get(0xc000149ae0, 0x12a05f200)
/usr/local/go/src/regexp/regexp.go:255 +0x21a
regexp.(*Regexp).doExecute(0xc000149ae0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc004aa8000, 0xdb4, 0xb0b, 0x2, ...)
/usr/local/go/src/regexp/exec.go:538 +0xb5
regexp.(*Regexp).replaceAll(0xc000149ae0, 0x0, 0x0, 0x0, 0xc004aa8000, 0xdb4, 0x2, 0xc002699a58, 0xaa3a2a, 0x4, ...)
/usr/local/go/src/regexp/regexp.go:599 +0x1a5
regexp.(*Regexp).ReplaceAllString(0xc000149ae0, 0xc004aa8000, 0xdb4, 0x0, 0x0, 0xc00038e060, 0x127d7a0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
_/home/ec2-user/environment/cluster2/cluster2/base.Sanitize(0xaa3a2a, 0x4, 0xc004aa8000, 0xdb4, 0x0, 0x0, 0x10, 0xab52f4)
/home/ec2-user/environment/cluster2/cluster2/base/sanitizer.go:44 +0xc9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f920, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:304 +0x33a
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 55 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2080016c0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc003b1c380, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc003b1c370, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0011af800, 0x7d4, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000c501c0, 0x3a, 0xc0011af800, 0x7d4, 0xc0011af800, 0x7d4, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000c501c0, 0x3a, 0xc0011af800, 0x7d4, 0xaa0378, 0x1, 0xc0011af800, 0x7d4, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f9e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 56 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe15c001080, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00107eb10, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00107eb00, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001342b00, 0x535, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000ccdd40, 0x40, 0xc001342b00, 0x535, 0xc001342b00, 0x535, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000ccdd40, 0x40, 0xc001342b00, 0x535, 0xaa0378, 0x1, 0xc001342b00, 0x535, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007faa0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 57 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc005094000, 0x74b, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc005094000, 0x74b, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc005094000, 0x74b, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc005094000, 0x74b, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc005094000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc005094000, 0x74b, 0x0, 0x0, 0x74b, 0x900, 0xc005094000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc005094000, 0x74b, 0x74b, 0x900)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000b72ec0, 0x3c, 0xc005094000, 0x74b, 0xc005094000, 0x74b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000b72ec0, 0x3c, 0xc005094000, 0x74b, 0xaa0378, 0x1, 0xc005094000, 0x74b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007fb60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 58 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe188013ec0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00241ccc0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00241ccb0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001493500, 0x90f, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000e7b0e0, 0x2d, 0xc001493500, 0x90f, 0xc001493500, 0x90f, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000e7b0e0, 0x2d, 0xc001493500, 0x90f, 0xaa0378, 0x1, 0xc001493500, 0x90f, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007fc20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 59 [semacquire]:
github.com/jaytaylor/html2text.FromString(0xc002a93500, 0x1352, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc0016185d0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:102 +0x42
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000e1f518, 0xc0009c72c0, 0xc0050486e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007fce0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 60 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc001bf2000, 0x9a1, 0xc0007ad218, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc001bf2000, 0x9a1, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0007ad3c0, 0xaa04fd, 0x2, 0xc001bf2000, 0x9a1, 0x1, 0x203, 0xc005bcfca0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001bf2000, 0x9a1, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0007ad518, 0xc0010b0340, 0x37, 0xc001287840)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007fda0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 61 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00229cf80, 0xc001983210, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001983588, 0xaa04f9, 0x2, 0xc003441180, 0x368, 0x7, 0xc001983290, 0x42d308)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0x2, 0xc004d5dc50)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:635 +0x82
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc004d5db00, 0xc001983320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc001983518, 0xc0011aa180, 0xc00229dba0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc001983518, 0xc0011aa180, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc001983518, 0xc0011aa180, 0xc0011aa180)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007fe60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 62 [semacquire]:
encoding/json.Unmarshal(0xc005ba88c0, 0x124, 0x140, 0x9cfa40, 0xc00522caa0, 0x140, 0xaa366e)
/usr/local/go/src/encoding/json/decode.go:100 +0x31
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.(*Model).Predict(0xc00522ca20, 0xc001285880, 0x32e, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:153 +0x16a
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001285880, 0x32e, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:31 +0xa0
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000efff40, 0x29, 0xc001285880, 0x32e, 0xc001285880, 0x32e, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000efff40, 0x29, 0xc001285880, 0x32e, 0xaa0378, 0x1, 0xc001285880, 0x32e, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007ff20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 63 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_Predict(0x7fe1f41bcff0, 0x7fe1957cd1d0, 0x7fe194223f70)
_cgo_gotypes.go:93 +0x4e
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.(*Model).Predict.func2(0xc00642bc00, 0x7fe1957cd1d0, 0x7fe1957cd1d0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:144 +0x60
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.(*Model).Predict(0xc00642bc00, 0xc0041222c0, 0x299, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:144 +0xa8
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0041222c0, 0x299, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:31 +0xa0
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002388280, 0x1d, 0xc0041222c0, 0x299, 0xc0041222c0, 0x299, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002388280, 0x1d, 0xc0041222c0, 0x299, 0xaa0378, 0x1, 0xc0041222c0, 0x299, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356000, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 65 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc0047e2c80, 0xc0008572c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0008573c0, 0xaa04fd, 0x2, 0xc0007c0a00, 0x1435, 0x64, 0x3da, 0xc003212040)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0007c0a00, 0x1435, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000857518, 0xc00103fce0, 0x3c, 0xc000efe040)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356180, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 66 [runnable]:
regexp.(*Regexp).doExecute(0xc000149400, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc001297210, 0x7, 0x0, 0x2, ...)
/usr/local/go/src/regexp/exec.go:521 +0x59b
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc001297210, 0x7, 0x2, 0xc0069da1d0, 0x0, 0xc0069da270, ...)
/usr/local/go/src/regexp/regexp.go:599 +0x1a5
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc001297210, 0x7, 0xaa0378, 0x1, 0xc005ab7e80, 0x5)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0069da3d8, 0xc005107110, 0xc001cee210, 0xc0069da308)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069da3d8, 0xc0051070a0, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0069db0c0, 0xc0051070a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:155 +0x7cc
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0069db0c0, 0xc0051070a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069db0c0, 0xc005106fc0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0008590c0, 0xc005106fc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0008590c0, 0xc005106fc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069db0c0, 0xc005106ee0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0008590c0, 0xc005106ee0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0008590c0, 0xc005106ee0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069db0c0, 0xc0051067e0, 0xc002c4a700, 0x7fe260fee2f0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0008590c0, 0xc0051067e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0008590c0, 0xc0051067e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069db0c0, 0xc005106700, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0008590c0, 0xc005106700, 0x54b170, 0xc002ca1ee0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc005106700, 0x0, 0x0, 0x0, 0x0, 0xc005106700, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc00566bfb0, 0x0, 0x0, 0x0, 0x1500, 0xc00240c000, 0x13e8, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc00240c000, 0x13e8, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc001217f10)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000859518, 0xc0013973a0, 0xc002489120)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356240, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 67 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe210000c60, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000dfb780, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc000dfb770, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001077500, 0x9a0, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0001140c0, 0x29, 0xc001077500, 0x9a0, 0xc001077500, 0x9a0, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0001140c0, 0x29, 0xc001077500, 0x9a0, 0xaa0378, 0x1, 0xc001077500, 0x9a0, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 68 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc00162c000, 0x57b, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc00162c000, 0x57b, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc00162c000, 0x57b, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc00162c000, 0x57b, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc00162c000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00162c000, 0x57b, 0x0, 0x0, 0x57b, 0xa80, 0xc00162c000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00162c000, 0x57b, 0x57b, 0xa80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000b72100, 0x3c, 0xc00162c000, 0x57b, 0xc00162c000, 0x57b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000b72100, 0x3c, 0xc00162c000, 0x57b, 0xaa0378, 0x1, 0xc00162c000, 0x57b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003563c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 69 [semacquire]:
github.com/gigawattio/window.Rolling(0xc0035dc500, 0x242, 0x242, 0x3, 0x0, 0x241, 0xc0035dc500)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00171d518, 0xc0010b1fc0, 0xc0010b00a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 70 [semacquire]:
bytes.makeSlice(0x11c, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc002d1f220, 0x9c, 0x40)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc002d1f220, 0xc000c59a40, 0x9c, 0x5165d7, 0xc004396fce, 0xc0003ce7b6)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/advancedlogic/GoOse.(*outputFormatter).Text.func1(0xc0005638f0)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:103 +0x93
github.com/advancedlogic/GoOse.(*outputFormatter).Text.func1(0xc000563880)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:107 +0x5b
github.com/advancedlogic/GoOse.(*outputFormatter).Text.func1(0xc000563260)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:107 +0x5b
github.com/advancedlogic/GoOse.(*outputFormatter).Text(0xc002d1f388, 0xc004c3b0e0, 0x6, 0xc0012833b0)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:112 +0xbd
github.com/advancedlogic/GoOse.(*outputFormatter).getOutputText(0xc002d1f388, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:120 +0x48
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc002d1f388, 0xc004c3b0e0, 0xaa04f9, 0x2, 0x42d308, 0x8, 0x9ecc80, 0xc002d1f301, 0xc004f5fdb8)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:45 +0x103
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc002d1f518, 0xc004c3b0e0, 0xaa04f9, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356540, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 71 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc001d1d500, 0x1331, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:81 +0x208
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc001d1d500, 0x1331, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc001d1d500, 0x1331, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc001d1d500, 0x1331, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc001d1d500)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc001d1d500, 0x1331, 0x0, 0x0, 0x1331, 0x1500, 0xc001d1d500)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc001d1d500, 0x1331, 0x1331, 0x1500)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000dac630, 0x2d, 0xc001d1d500, 0x1331, 0xc001d1d500, 0x1331, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000dac630, 0x2d, 0xc001d1d500, 0x1331, 0xaa0378, 0x1, 0xc001d1d500, 0x1331, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 72 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc0033a80d0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:901 +0x820
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0033a80d0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0033a80d0, 0xc0054a5180, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7b620, 0xc000e8b560, 0x0, 0x0, 0x0, 0x0, 0xb6bf7f, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc001b4c450, 0x0, 0x0, 0x0, 0x1000, 0xc003a41000, 0xea3, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:93 +0x76
github.com/jaytaylor/html2text.FromString(0xc003a41000, 0xea3, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc001c87480)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000e2d518, 0xc00039eba0, 0xc0004e6dc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003566c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 73 [runnable]:
regexp.(*inputString).step(0xc001aa0550, 0xad, 0xffffffffffffffff, 0x1)
/usr/local/go/src/regexp/regexp.go:365 +0xbc
regexp.(*Regexp).tryBacktrack(0xc000149400, 0xc001aa04d0, 0xb886e0, 0xc001aa0550, 0xc000000001, 0xad, 0x0)
/usr/local/go/src/regexp/backtrack.go:209 +0x984
regexp.(*Regexp).backtrack(0xc000149400, 0x0, 0x0, 0x0, 0xc0049eb880, 0xd2, 0xad, 0x2, 0xc006288970, 0x0, ...)
/usr/local/go/src/regexp/backtrack.go:353 +0x39d
regexp.(*Regexp).doExecute(0xc000149400, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0049eb880, 0xd2, 0xab, 0x2, ...)
/usr/local/go/src/regexp/exec.go:535 +0x453
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc0049eb880, 0xd2, 0x2, 0xc0023cc198, 0xc00000000a, 0x2, ...)
/usr/local/go/src/regexp/regexp.go:599 +0x1a5
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc0049eb880, 0xd2, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0023cd0c0, 0xc00282d2d0, 0xc0023cc1b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0023cd0c0, 0xc00282d260, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0023cd0c0, 0xc00282d260, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0023cd0c0, 0xc00282d260, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0023cd0c0, 0xc00282d260, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0023cd0c0, 0xc00282c5b0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0007cb0c0, 0xc00282c5b0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007cb0c0, 0xc00282c5b0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0023cd0c0, 0xc00282c4d0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0007cb0c0, 0xc00282c4d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007cb0c0, 0xc00282c4d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0023cd0c0, 0xc00543f570, 0xc0007cad40, 0x46a81e)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0007cb0c0, 0xc00543f570, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007cb0c0, 0xc00543f570, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0023cd0c0, 0xc00543f490, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007cb0c0, 0xc00543f490, 0x54b170, 0xc002d2b380)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc00543f490, 0x0, 0x0, 0x0, 0x0, 0xc00543f490, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc001cfeff0, 0x0, 0x0, 0x0, 0xc00, 0xc00376ec00, 0xba6, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc00376ec00, 0xba6, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc0003cec30)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0007cb518, 0xc001396b60, 0xc0032124e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356780, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 74 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc001aba400, 0x85a, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:81 +0x208
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc001aba400, 0x85a, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc001aba400, 0x85a, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc001aba400, 0x85a, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc001aba400, 0x85a, 0x0, 0x0, 0x85a, 0xc001aba400, 0x85a)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc001aba400, 0x85a, 0x85a, 0x900)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000c50b80, 0x33, 0xc001aba400, 0x85a, 0xc001aba400, 0x85a, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000c50b80, 0x33, 0xc001aba400, 0x85a, 0xaa0378, 0x1, 0xc001aba400, 0x85a, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356840, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 75 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2041597b0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00124c2d0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00124c2c0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0017d4a80, 0x945, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000426960, 0x57, 0xc0017d4a80, 0x945, 0xc0017d4a80, 0x945, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000426960, 0x57, 0xc0017d4a80, 0x945, 0xaa0378, 0x1, 0xc0017d4a80, 0x945, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356900, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 76 [runnable]:
sync.(*RWMutex).RUnlock(0xc00039e108)
/usr/local/go/src/sync/rwmutex.go:62 +0x50
github.com/fatih/set.(*Set).Has(0xc00039e100, 0xc0014bd2c0, 0x1, 0x1, 0xc000480001)
/home/develop/go/src/github.com/fatih/set/set_ts.go:86 +0x10c
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0014bd3c0, 0xaa0503, 0x2, 0xc0010eec00, 0x3ef, 0x3, 0xd4, 0xc0034d1e00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:81 +0x1e7
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0010eec00, 0x3ef, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0014bd518, 0xc001246100, 0x3a, 0xc000a9ea00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003569c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 77 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2340a1280, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000351140, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc000351100, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002923880, 0x376, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000f62a50, 0x44, 0xc002923880, 0x376, 0xc002923880, 0x376, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000f62a50, 0x44, 0xc002923880, 0x376, 0xaa0378, 0x1, 0xc002923880, 0x376, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356a80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 78 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe21c001600, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0001b6e00, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0001b6df0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00292c480, 0x228, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0017329c0, 0x32, 0xc00292c480, 0x228, 0xc00292c480, 0x228, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0017329c0, 0x32, 0xc00292c480, 0x228, 0xaa0378, 0x1, 0xc00292c480, 0x228, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356b40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 79 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1e41be320, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0038ee780, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0038ee770, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00152e000, 0x1cf5, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00087a340, 0x3e, 0xc00152e000, 0x1cf5, 0xc00152e000, 0x1cf5, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00087a340, 0x3e, 0xc00152e000, 0x1cf5, 0xaa0378, 0x1, 0xc00152e000, 0x1cf5, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356c00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 80 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc006350000, 0x71e, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:71 +0x129
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc006350000, 0x71e, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc006350000, 0x71e, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc006350000, 0x71e, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc006350000, 0x71e, 0x0, 0x0, 0x71e, 0xc006350000, 0x71e)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc006350000, 0x71e, 0x71e, 0x900)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000b6c330, 0x26, 0xc006350000, 0x71e, 0xc006350000, 0x71e, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000b6c330, 0x26, 0xc006350000, 0x71e, 0xaa0378, 0x1, 0xc006350000, 0x71e, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356cc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 81 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc003279320, 0x4000000000000000, 0x4019000000000000)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc003145518, 0xc0010b0140, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:423 +0x38b
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356d80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 82 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000f13588, 0xaa04f9, 0x2, 0xc003959c20, 0x116, 0x1, 0xb, 0xc0024882c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc000f13518, 0xc001cfc0c0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356e40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 83 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe234004ba0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0021eaf00, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0021eaef0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000fd8f00, 0x4cb, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0011262d0, 0x42, 0xc000fd8f00, 0x4cb, 0xc000fd8f00, 0x4cb, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0011262d0, 0x42, 0xc000fd8f00, 0x4cb, 0xaa0378, 0x1, 0xc000fd8f00, 0x4cb, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356f00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 84 [semacquire]:
strings.Replace(0xc0006e3700, 0x536, 0xaa04bd, 0x2, 0xaa03a3, 0x1, 0x1, 0xb7b900, 0xc00005e060)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/jaytaylor/html2text.FromHTMLNode(0xc000ad9c00, 0x0, 0x0, 0x0, 0x0, 0xc000ad9c00, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:81 +0x158
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc000adab10, 0x0, 0x0, 0x0, 0x900, 0xc000ad1600, 0x8e8, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc000ad1600, 0x8e8, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc000ad40d0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000ae3518, 0xc0007625c0, 0xc000762e00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000356fc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 85 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc004055560, 0xa9ece0, 0xc004055530)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00383d518, 0xc001397960, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:384 +0x144
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357080, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 86 [runnable]:
internal/reflectlite.Swapper.func9(0x6f, 0x6e)
/usr/local/go/src/internal/reflectlite/swapper.go:64 +0x10f
sort.insertionSort_func(0xc000a7b7d0, 0xc001b62060, 0x64, 0x78)
/usr/local/go/src/sort/zfuncversion.go:13 +0x50
sort.stable_func(0xc000a7b7d0, 0xc001b62060, 0x233)
/usr/local/go/src/sort/zfuncversion.go:167 +0x51
sort.SliceStable(0x9de7e0, 0xc00656ccc0, 0xc000a7b7d0)
/usr/local/go/src/sort/slice.go:27 +0xcb
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc005808000, 0x411, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:48 +0x1c8
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc005808000, 0x411, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc005808000, 0x411, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc005808000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc005808000, 0x411, 0x0, 0x0, 0x411, 0x600, 0xc005808000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc005808000, 0x411, 0x411, 0x600)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00002bdb0, 0x4a, 0xc005808000, 0x411, 0xc005808000, 0x411, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00002bdb0, 0x4a, 0xc005808000, 0x411, 0xaa0378, 0x1, 0xc005808000, 0x411, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357140, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 87 [semacquire]:
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc000c33300, 0xc000c332c8, 0xc000c33350, 0x42ca96, 0xc001999fb0, 0x30)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:824 +0x69
github.com/andybalholm/cascadia.ParseGroup(0xab6b38, 0x13, 0x1, 0xc000c33368, 0xc000c33380, 0x42d308, 0x30)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xab6b38, 0x13, 0x1, 0xb856e0, 0xc005bce5c0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0048d7f50, 0xab6b38, 0x13, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCanonicalLink(0xc000c33518, 0xc00135eb00, 0xaba6c1, 0x1a)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:239 +0x47
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:135 +0x446
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357200, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 88 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2340032e0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc003778110, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc003778100, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000484a00, 0x69a, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000ea65a0, 0x46, 0xc000484a00, 0x69a, 0xc000484a00, 0x69a, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000ea65a0, 0x46, 0xc000484a00, 0x69a, 0xaa0378, 0x1, 0xc000484a00, 0x69a, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003572c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 89 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc003738750, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0045e1518, 0xc001397620, 0x35, 0xc000efea40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357380, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 90 [semacquire]:
bytes.makeSlice(0x1e00, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc000c830e8, 0x200, 0x800)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).ReadFrom(0xc000c830e8, 0xb7b620, 0xc0032c64b0, 0x203000, 0x203000, 0x203000)
/usr/local/go/src/bytes/buffer.go:202 +0x48
io/ioutil.readAll(0xb7b620, 0xc0032c64b0, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:36 +0xe3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc0032c64b0, 0xc000c831d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0032c64b0, 0x0, 0x0, 0x0, 0x1000, 0xc0065bb000, 0xe46, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc0065bb000, 0xe46, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc0010dbd00)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000c83518, 0xc00135e220, 0xc0023002c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357440, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 91 [semacquire]:
regexp.(*Regexp).ReplaceAllString(0xc004b21900, 0xc002764e00, 0x344, 0xaa0378, 0x1, 0x1, 0x12)
/usr/local/go/src/regexp/regexp.go:560 +0x135
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000a79518, 0xc00000e140, 0xc001397260)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:319 +0x778
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357500, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 92 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2340a3100, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00279c150, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00279c140, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002d9e000, 0xed9, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000cd4150, 0x29, 0xc002d9e000, 0xed9, 0xc002d9e000, 0xed9, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000cd4150, 0x29, 0xc002d9e000, 0xed9, 0xaa0378, 0x1, 0xc002d9e000, 0xed9, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003575c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 93 [semacquire]:
regexp.makeOnePass.func1(0xc, 0xc0062ad7a0, 0xf, 0xf, 0x1)
/usr/local/go/src/regexp/onepass.go:385 +0xbdf
regexp.makeOnePass(0xc00568a120, 0xc00568a120)
/usr/local/go/src/regexp/onepass.go:448 +0x398
regexp.compileOnePass(0xc00568a0c0, 0xc00568a0c0)
/usr/local/go/src/regexp/onepass.go:500 +0x129
regexp.compile(0xabc10c, 0x11, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc00064d300, 0x0, 0xabc106, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc00064d300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc00064d300, 0x0, 0xb856e0, 0xc00039e188, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc00064d300, 0xc000412c40, 0x0, 0x8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc00064d300, 0x42ca96, 0xc00568a090, 0x30, 0x28, 0xa4b3a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xabc101, 0x1d, 0xc00064d370, 0x42d308, 0x30, 0xa4b3a0, 0xb85601)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xabc101, 0x1d, 0xc000fefe40, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc000adb530, 0xabc101, 0x1d, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc00064d518, 0xc000762ae0, 0xabc101, 0x1d, 0x34, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:133 +0x3a7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357680, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 95 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1f41c64c0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002fdf0c0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002fdf0b0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00170bc00, 0xad9, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000de6000, 0x76, 0xc00170bc00, 0xad9, 0xc00170bc00, 0xad9, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000de6000, 0x76, 0xc00170bc00, 0xad9, 0xaa0378, 0x1, 0xc00170bc00, 0xad9, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357800, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 97 [semacquire]:
github.com/gigawattio/window.Rolling(0xc002200000, 0x2b5, 0x2b5, 0x3, 0x0, 0x2b4, 0xc002200000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000497518, 0xc00039e9e0, 0xc000c45000)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357980, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 98 [semacquire]:
strings.genSplit(0xc004770000, 0x29ac, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0036a6000, 0x8e8, 0x8e8)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001b153c0, 0xaa04d9, 0x2, 0xc000abca80, 0x29ac, 0x8, 0x8e8, 0xc0033839c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000abca80, 0x29ac, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001b15518, 0xc003382920, 0x4c, 0xc000ea6c30)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357a40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 99 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc003e93500, 0x959, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc003e93500, 0x959, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc003e93500, 0x959, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc003e93500, 0x959, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc003e93500, 0x959, 0x0, 0x0, 0x959, 0xc003e93500, 0x959)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc003e93500, 0x959, 0x959, 0xa80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000aa8b80, 0x37, 0xc003e93500, 0x959, 0xc003e93500, 0x959, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000aa8b80, 0x37, 0xc003e93500, 0x959, 0xaa0378, 0x1, 0xc003e93500, 0x959, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357b00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 100 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc004bcb000, 0xad9, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc004bcb000, 0xad9, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc004bcb000, 0xad9, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc004bcb000, 0xad9, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc004bcb000, 0xad9, 0x0, 0x0, 0xad9, 0xc004bcb000, 0xad9)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc004bcb000, 0xad9, 0xad9, 0xc00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0001141b0, 0x2e, 0xc004bcb000, 0xad9, 0xc004bcb000, 0xad9, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0001141b0, 0x2e, 0xc004bcb000, 0xad9, 0xaa0378, 0x1, 0xc004bcb000, 0xad9, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357bc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 101 [semacquire]:
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc000579300, 0x0, 0xb856e0, 0xc00039e048, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:724 +0x716
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc000579300, 0xc000967b90, 0x0, 0x8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc000579300, 0x42ca96, 0xc0035d08a0, 0x30, 0x28, 0xa4b3a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xabc101, 0x1d, 0xc000579370, 0x42d308, 0x30, 0xa4b3a0, 0xb85601)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xabc101, 0x1d, 0xc001396ca0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc001099530, 0xabc101, 0x1d, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc000579518, 0xc0010b1da0, 0xabc101, 0x1d, 0x15, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:133 +0x3a7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357c80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 102 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1e828f490, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0003511d0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0003511b0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0008bd000, 0x761, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00235c0c0, 0x34, 0xc0008bd000, 0x761, 0xc0008bd000, 0x761, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00235c0c0, 0x34, 0xc0008bd000, 0x761, 0xaa0378, 0x1, 0xc0008bd000, 0x761, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357d40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 104 [semacquire]:
regexp.(*Regexp).expand(0xc000149400, 0xc004233100, 0x100, 0x100, 0xaa0378, 0x1, 0x0, 0x0, 0x0, 0xc000ca12c0, ...)
/usr/local/go/src/regexp/regexp.go:948 +0xa9b
regexp.(*Regexp).ReplaceAllString.func1(0xc004233100, 0x100, 0x100, 0xc00403e290, 0x2, 0x2, 0xc000ca12c0, 0x1c4, 0xfe)
/usr/local/go/src/regexp/regexp.go:558 +0xbf
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc000ca12c0, 0x1c4, 0x2, 0xc005026198, 0xc00403e280, 0x1, ...)
/usr/local/go/src/regexp/regexp.go:616 +0x2e9
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc000ca12c0, 0x1c4, 0xaa0378, 0x1, 0x1189d00, 0x1)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0050270c0, 0xc00663a620, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0050270c0, 0xc004a509a0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0050270c0, 0xc004a509a0, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0050270c0, 0xc004a509a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0050270c0, 0xc004a509a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0050270c0, 0xc004a50380, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000e090c0, 0xc004a50380, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000e090c0, 0xc004a50380, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0050270c0, 0xc004a502a0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000e090c0, 0xc004a502a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000e090c0, 0xc004a502a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0050270c0, 0xc001701b90, 0xc002c4a700, 0x7fe260fee2f0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000e090c0, 0xc001701b90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000e090c0, 0xc001701b90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0050270c0, 0xc001701ab0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000e090c0, 0xc001701ab0, 0x54b170, 0xc003d5d110)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc001701ab0, 0x0, 0x0, 0x0, 0x0, 0xc001701ab0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0000aced0, 0x0, 0x0, 0x0, 0x1300, 0xc004baa000, 0x11d5, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc004baa000, 0x11d5, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc001195a00)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000e09518, 0xc0000b25e0, 0xc005c547e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357ec0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 105 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1840056e0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0038ee040, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0038ee030, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001918d00, 0x837, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0026dce10, 0x4d, 0xc001918d00, 0x837, 0xc001918d00, 0x837, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0026dce10, 0x4d, 0xc001918d00, 0x837, 0xaa0378, 0x1, 0xc001918d00, 0x837, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000357f80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 106 [runnable]:
internal/reflectlite.Swapper.func9(0x2e, 0x56)
/usr/local/go/src/internal/reflectlite/swapper.go:64 +0x10f
sort.swapRange_func(0xc0016ff7d0, 0xc005312ea0, 0x28, 0x50, 0x28)
/usr/local/go/src/sort/zfuncversion.go:67 +0x4b
sort.rotate_func(0xc0016ff7d0, 0xc005312ea0, 0x28, 0x50, 0x78)
/usr/local/go/src/sort/zfuncversion.go:264 +0x104
sort.symMerge_func(0xc0016ff7d0, 0xc005312ea0, 0x0, 0x50, 0xa0)
/usr/local/go/src/sort/zfuncversion.go:241 +0x364
sort.stable_func(0xc0016ff7d0, 0xc005312ea0, 0x1b6)
/usr/local/go/src/sort/zfuncversion.go:175 +0xc2
sort.SliceStable(0x9de7e0, 0xc0058345e0, 0xc0022c37d0)
/usr/local/go/src/sort/slice.go:27 +0xcb
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc00016ee00, 0x36d, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:48 +0x1c8
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc00016ee00, 0x36d, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc00016ee00, 0x36d, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00016ee00, 0x36d, 0x0, 0x0, 0x36d, 0xc00016ee00, 0x36d)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00016ee00, 0x36d, 0x36d, 0x380)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0007640f0, 0x48, 0xc00016ee00, 0x36d, 0xc00016ee00, 0x36d, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0007640f0, 0x48, 0xc00016ee00, 0x36d, 0xaa0378, 0x1, 0xc00016ee00, 0x36d, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376060, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 107 [semacquire]:
bytes.makeSlice(0x1ba4, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc006612330, 0xf4, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc006612330, 0xc0063e2900, 0xf4, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc006612330, 0xc0063e2900, 0xf4, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc006612330, 0xc006614000, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc006612330, 0xc0063fdf80, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc006612330, 0xc0063fddc0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc006612330, 0xc004078930, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc006612330, 0xc004078850, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc006612330, 0xc004078150, 0xb84d20, 0xc006612330)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc006612330, 0xc004078150, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc006612180, 0x161f00eb0, 0x1251a20, 0xc0063e6580, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376120, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 108 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc003fd2000, 0x970, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc003fd2000, 0x970, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc003fd2000, 0x970, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc003fd2000, 0x970, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc003fd2000, 0x970, 0x0, 0x0, 0x970, 0xc003fd2000, 0x970)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc003fd2000, 0x970, 0x970, 0xc00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000b64780, 0x7d, 0xc003fd2000, 0x970, 0xc003fd2000, 0x970, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000b64780, 0x7d, 0xc003fd2000, 0x970, 0xaa0378, 0x1, 0xc003fd2000, 0x970, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003761e0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 109 [semacquire]:
strings.genSplit(0xc00325e000, 0x2f6b, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0025e8000, 0xae0, 0xae0)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0016073c0, 0xaa059b, 0x2, 0xc0025d1000, 0x2f6b, 0x21, 0xae0, 0xc00137f440)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0025d1000, 0x2f6b, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001607518, 0xc00039eb60, 0x64, 0xc000f07110)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003762a0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 110 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc0032a89e0, 0xc0011b3220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0011b3588, 0xaa04f9, 0x2, 0xc003276380, 0xd6d, 0x6, 0x1c, 0xc00229c2a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0011b3518, 0xc0009c6020, 0x93)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376360, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 111 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc000f3a000, 0x1f29, 0xc000cf3218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc000f3a000, 0x1f29, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000cf33c0, 0xaa04fd, 0x2, 0xc000f3a000, 0x1f29, 0x1, 0x6ef, 0xc0024891c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000f3a000, 0x1f29, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000cf3518, 0xc000762040, 0x54, 0xc00042606f)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376420, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 112 [semacquire]:
bytes.makeSlice(0x16aa, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc004429138, 0xca, 0x536)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc004429138, 0xc001112000, 0xca, 0x5165d7, 0xc0017742d2, 0xc000772020)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00110c230)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00110c1c0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00110c000)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000f09260)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc005b1f950, 0xaa03ab, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).isHighLinkDensity(0xc004429518, 0xc005b1f950, 0x2)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:566 +0x95
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc004429518, 0xc00039eaa0, 0x58)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:386 +0x1ae
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003764e0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 113 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc001001500, 0xc0007872c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0007873c0, 0xaa0569, 0x2, 0xc001b82a80, 0x91b, 0x30, 0x1d5, 0xc003308160)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001b82a80, 0x91b, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000787518, 0xc002300080, 0x60, 0xc00068d7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003765a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 114 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1b8000da0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001618620, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001618600, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000e83b80, 0x270, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000bba0f0, 0x49, 0xc000e83b80, 0x270, 0xc000e83b80, 0x270, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000bba0f0, 0x49, 0xc000e83b80, 0x270, 0xaa0378, 0x1, 0xc000e83b80, 0x270, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376660, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 115 [semacquire]:
strings.genSplit(0xc002ad0600, 0x1866, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc00302b800, 0x46d, 0x46d)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00043d3c0, 0xaa0599, 0x2, 0xc0024b6600, 0x1866, 0x1, 0x46d, 0xc001c5ea40)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0024b6600, 0x1866, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00043d518, 0xc0010b0b60, 0x33, 0xc00087a640)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376720, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 116 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe23020c1a0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002b43070, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002b43060, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003310600, 0x5cd, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00002a653, 0x3b, 0xc003310600, 0x5cd, 0xc003310600, 0x5cd, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00002a653, 0x3b, 0xc003310600, 0x5cd, 0xaa0378, 0x1, 0xc003310600, 0x5cd, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003767e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 117 [runnable]:
github.com/fatih/set.(*Set).Add(0xc0059657a0, 0xc001ef5220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001ef5588, 0xaa04f9, 0x2, 0xc005172c00, 0xbb5, 0x25, 0x62, 0xc0063e6b00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc001ef5518, 0xc001246a40, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003768a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 118 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000b2b3c0, 0xaa0503, 0x2, 0xc000e4b800, 0x1548, 0x4, 0x401, 0xc0004e6040)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000e4b800, 0x1548, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000b2b518, 0xc001396f80, 0x2e, 0xc000114570)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376960, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 119 [semacquire]:
regexp/syntax.(*compiler).inst(...)
/usr/local/go/src/regexp/syntax/compile.go:173
regexp/syntax.(*compiler).rune(0xc0008d2e38, 0xc003d6c2f8, 0x1, 0x2, 0xd5, 0xe00000007)
/usr/local/go/src/regexp/syntax/compile.go:267 +0x27c
regexp/syntax.(*compiler).compile(0xc0008d2e38, 0xc003cec2a0, 0x200000001)
/usr/local/go/src/regexp/syntax/compile.go:110 +0x12c
regexp/syntax.(*compiler).compile(0xc0008d2e38, 0xc003cec310, 0x0)
/usr/local/go/src/regexp/syntax/compile.go:156 +0xb29
regexp/syntax.Compile(0xc003cec310, 0xc003cec310, 0x1, 0x1)
/usr/local/go/src/regexp/syntax/compile.go:83 +0x10a
regexp.compile(0xaba6cc, 0xe, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:178 +0xd7
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0008d3300, 0x0, 0xaba6c6, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0008d3300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0008d3300, 0x0, 0x0, 0x0, 0xb856e0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0008d3300, 0x75a856, 0x0, 0xc0007511f0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0008d3300, 0xc0008d32c8, 0xc0008d3350, 0x42ca96, 0xc001914540, 0x30)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xaba6c1, 0x1a, 0x1, 0xc0008d3368, 0xc0008d3380, 0x42d308, 0x30)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xaba6c1, 0x1a, 0x1, 0xb856e0, 0xc00608ce00)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc000bfc1e0, 0xaba6c1, 0x1a, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc0008d3518, 0xc000b32920, 0xaba6c1, 0x1a, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:134 +0x401
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376a20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 121 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00608d000, 0xc0011072c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0011073c0, 0xaa04d9, 0x2, 0xc0008de480, 0x19f1, 0x5, 0x4c1, 0xc00229d360)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0008de480, 0x19f1, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001107518, 0xc0018848c0, 0x3c, 0xc000c50a40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376ba0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 122 [semacquire]:
regexp.onePassCopy(0xc0010989f0, 0xc0010989f0)
/usr/local/go/src/regexp/onepass.go:225 +0x5d
regexp.compileOnePass(0xc0010989f0, 0xc0010989f0)
/usr/local/go/src/regexp/onepass.go:497 +0x11b
regexp.compile(0xabc10c, 0x11, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc001183300, 0x0, 0xabc106, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc001183300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc001183300, 0x0, 0xb856e0, 0xc00039e048, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc001183300, 0xc000f653b0, 0x0, 0x8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc001183300, 0x42ca96, 0xc004d5df50, 0x30, 0x28, 0xa4b3a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xabc101, 0x1d, 0xc001183370, 0x42d308, 0x30, 0xa4b3a0, 0xb85601)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xabc101, 0x1d, 0xc00229d4a0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc00040de00, 0xabc101, 0x1d, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc001183518, 0xc00039ea40, 0xabc101, 0x1d, 0x24, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:133 +0x3a7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376c60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 123 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003491588, 0xaa04f9, 0x2, 0xc0027baf20, 0x151, 0x0, 0xc0034913d8, 0xc0027efa08)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc003491518, 0xc0034ac0f0, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:528 +0xfd
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc003491518, 0xc000fee620, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376d20, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 124 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc004b61588, 0xaa04f9, 0x2, 0xc0053edb80, 0x12a, 0x23, 0x7b, 0xc00522c9a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc004b61518, 0xc0010b1e00, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376de0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 125 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00229dde0, 0xc0004392c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0004393c0, 0xaa04f9, 0x2, 0xc001b2a000, 0x82b, 0x4, 0x1b5, 0xc001397760)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001b2a000, 0x82b, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000439518, 0xc0010b10c0, 0x4d, 0xc001126140)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376ea0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 126 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc001258a80, 0x9b8, 0xc000eb5218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc001258a80, 0x9b8, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000eb53c0, 0xaa059b, 0x2, 0xc001258a80, 0x9b8, 0x17, 0x200, 0xc001c5f060)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001258a80, 0x9b8, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000eb5518, 0xc000feffa0, 0x45, 0xc000f620a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376f60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 127 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc001e88000, 0x7c8, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:81 +0x208
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc001e88000, 0x7c8, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc001e88000, 0x7c8, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc001e88000, 0x7c8, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc001e88000, 0x7c8, 0x0, 0x0, 0x7c8, 0xc001e88000, 0x7c8)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc001e88000, 0x7c8, 0x7c8, 0x800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0008960c0, 0x53, 0xc001e88000, 0x7c8, 0xc001e88000, 0x7c8, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0008960c0, 0x53, 0xc001e88000, 0x7c8, 0xaa0378, 0x1, 0xc001e88000, 0x7c8, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377020, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 128 [semacquire]:
github.com/gigawattio/window.Rolling(0xc004dc0000, 0x3ac, 0x3ac, 0x3, 0x0, 0x3ab, 0xc004dc0000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000437518, 0xc000fefd40, 0xc0024889e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003770e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 129 [semacquire]:
github.com/gigawattio/window.Rolling(0xc00397c000, 0x3ff, 0x3ff, 0x3, 0x0, 0x3fe, 0xc00397c000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000b2d518, 0xc0009c79e0, 0xc0010b01c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003771a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 130 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc0033f1000, 0x7ec, 0xc000f79218, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc0033f1000, 0x7ec, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000f793c0, 0xaa059b, 0x2, 0xc0033f1000, 0x7ec, 0x11, 0x275, 0xc003212060)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0033f1000, 0x7ec, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000f79518, 0xc0010b0e20, 0x36, 0xc00087b39c)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377260, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 131 [semacquire]:
github.com/jaytaylor/html2text.FromString(0xc0065d5500, 0x1415, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000a9e6c0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:102 +0x42
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001495518, 0xc001000360, 0xc004cda600)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377320, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 132 [semacquire]:
github.com/gigawattio/window.Rolling(0xc0027b0c80, 0xc4, 0xc4, 0x3, 0x0, 0xc3, 0xc0027b0c80)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000777518, 0xc000fefa80, 0xc0032a9bc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003773e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 133 [semacquire]:
github.com/PuerkitoBio/goquery.newSingleSelection(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:113
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc00102e3f0, 0xc000695408, 0xc002c71470)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x9b
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc000695518, 0xc002c71470, 0xc002c71470)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:720 +0x88
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003774a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 134 [semacquire]:
bytes.makeSlice(0x1e00, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0004af0e8, 0x200, 0x800)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).ReadFrom(0xc0004af0e8, 0xb7b620, 0xc005f56ab0, 0x203001, 0x203001, 0x203001)
/usr/local/go/src/bytes/buffer.go:202 +0x48
io/ioutil.readAll(0xb7b620, 0xc005f56ab0, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:36 +0xe3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc005f56ab0, 0xc0004af1d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc005f56ab0, 0x0, 0x0, 0x0, 0x1980, 0xc002f5ac80, 0x182d, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc002f5ac80, 0x182d, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000cd4870)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0004af518, 0xc0009c7c40, 0xc005bcf8e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377560, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 135 [semacquire]:
bytes.makeSlice(0xbf6, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc004ebd530, 0xaa, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc004ebd530, 0xc0011218c0, 0xaa, 0x6, 0xffffffffffffffff, 0xad0184)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc004ebd530, 0xc0011218c0, 0xaa, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc004ebd530, 0xc000fd6d20, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc004ebd530, 0xc000fd6a10, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc004ebd530, 0xc000f9bc70, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc004ebd530, 0xc000f9bb90, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc004ebd530, 0xc000f9b490, 0xb84d20, 0xc004ebd530)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc004ebd530, 0xc000f9b490, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc001340c60, 0x50d2a1, 0xc00112e6c0, 0x31, 0xaada21)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001569518, 0xc00135e680, 0xc005048a20)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:285 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377620, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 136 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1d82389f0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0028a6200, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0028a61f0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000a25100, 0x675, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00007e120, 0x52, 0xc000a25100, 0x675, 0xc000a25100, 0x675, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00007e120, 0x52, 0xc000a25100, 0x675, 0xaa0378, 0x1, 0xc000a25100, 0x675, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003776e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 137 [semacquire]:
strings.genSplit(0xc0055d4900, 0x884, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc002dd4000, 0x1d4, 0x1d4)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00077b3c0, 0xaa0599, 0x2, 0xc000a4e400, 0x884, 0x1, 0x1d4, 0xc001000320)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000a4e400, 0x884, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00077b518, 0xc00000e6e0, 0x31, 0xc000c502c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003777a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 138 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005bcfb40, 0xc0018e12c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0018e13c0, 0xaa05c9, 0x2, 0xc001a14000, 0x927, 0x6, 0x1fb, 0xc00137f9e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001a14000, 0x927, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0018e1518, 0xc000b326a0, 0x70, 0xc000dea5c7)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377860, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 139 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005049c60, 0xc00043b2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00043b3c0, 0xaa0503, 0x2, 0xc0007fbb00, 0xd3d, 0x4, 0x28e, 0xc00135fc60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0007fbb00, 0xd3d, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00043b518, 0xc00103f1a0, 0x46, 0xc001127a40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377920, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 140 [semacquire]:
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc0053ea480, 0x10a, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:560 +0x135
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc005fb50c0, 0xc0012139d0, 0xc005fb41b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005fb50c0, 0xc001213960, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc005fb50c0, 0xc001213960, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc005fb50c0, 0xc001213960, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc005fb50c0, 0xc001213960, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005fb50c0, 0xc000f300e0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0007790c0, 0xc000f300e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007790c0, 0xc000f300e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005fb50c0, 0xc002d89f10, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0007790c0, 0xc002d89f10, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007790c0, 0xc002d89f10, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005fb50c0, 0xc002d89810, 0xc003be6380, 0x7fe260fedc28)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0007790c0, 0xc002d89810, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007790c0, 0xc002d89810, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005fb50c0, 0xc002d89730, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007790c0, 0xc002d89730, 0x54b170, 0xc003ed1a00)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc002d89730, 0x0, 0x0, 0x0, 0x0, 0xc002d89730, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc002c8ec30, 0x0, 0x0, 0x0, 0x1000, 0xc003bd2000, 0xfe3, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc003bd2000, 0xfe3, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000b72580)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000779518, 0xc000762f00, 0xc001246840)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003779e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 141 [runnable]:
sync.(*RWMutex).Lock(0xc00000e368)
/usr/local/go/src/sync/rwmutex.go:92 +0xa3
github.com/fatih/set.(*Set).Add(0xc00000e360, 0xc0007472c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:31 +0x58
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0007473c0, 0xaa058b, 0x2, 0xc001853900, 0x11f7, 0x5, 0x39c, 0xc00135e020)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001853900, 0x11f7, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000747518, 0xc00229db20, 0x59, 0xc000896000)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377aa0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 142 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0014f6000, 0x642, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:59 +0x21d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0014f6000, 0x642, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0014f6000, 0x642, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0014f6000, 0x642, 0x0, 0x0, 0x642, 0xc0014f6000, 0x642)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0014f6000, 0x642, 0x642, 0xc00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000ccce40, 0x20, 0xc0014f6000, 0x642, 0xc0014f6000, 0x642, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000ccce40, 0x20, 0xc0014f6000, 0x642, 0xaa0378, 0x1, 0xc0014f6000, 0x642, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377b60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 143 [runnable]:
github.com/gigawattio/window.Rolling(0xc00261ea80, 0x299, 0x299, 0x3, 0x0, 0x298, 0xc00261ea80)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0008d5518, 0xc002409f60, 0xc001c5e840)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377c20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 144 [semacquire]:
bytes.makeSlice(0x2fc8, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc003cbf388, 0x90, 0x90)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc003cbf388, 0xc001022fc0, 0x90, 0x5165d7, 0xc00265c789, 0xc0028f8c30)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc005b165b0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc005b16540)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0022072d0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000aa5f80)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc001946150)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc00566b9e0, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003cbf518, 0xc0059f4420, 0x52, 0xc001fffe00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377ce0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 145 [semacquire]:
regexp.(*Regexp).replaceAll(0xc003c69400, 0x0, 0x0, 0x0, 0xc00184a600, 0x12d7, 0x2, 0xc002a83218, 0xc00184a600, 0x12d7, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc003c69400, 0xc00184a600, 0x12d7, 0xaa0378, 0x1, 0x1, 0x2c)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002a83518, 0xc00000eae0, 0xc00608d6a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:319 +0x778
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377da0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 146 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0017eaa00, 0x1462, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:71 +0x129
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0017eaa00, 0x1462, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0017eaa00, 0x1462, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0017eaa00, 0x1462, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0017eaa00, 0x1462, 0x0, 0x0, 0x1462, 0xc0017eaa00, 0x1462)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0017eaa00, 0x1462, 0x1462, 0x1800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001126332, 0x3e, 0xc0017eaa00, 0x1462, 0xc0017eaa00, 0x1462, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001126332, 0x3e, 0xc0017eaa00, 0x1462, 0xaa0378, 0x1, 0xc0017eaa00, 0x1462, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377e60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 147 [semacquire]:
github.com/PuerkitoBio/goquery.appendWithoutDuplicates(0xc004a07b00, 0x9, 0x10, 0xc0012f3d00, 0x15, 0x20, 0xc00042d020, 0xc0015c3390, 0x1, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/utilities.go:138 +0x172
github.com/PuerkitoBio/goquery.mapNodes(0xc002e2ce58, 0x1, 0x1, 0xc00042d0f0, 0x7fe260fee2f0, 0x0, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:694 +0x159
github.com/PuerkitoBio/goquery.getSiblingNodes(0xc002e2ce58, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0015c3390, ...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:598 +0x101
github.com/PuerkitoBio/goquery.(*Selection).Siblings(0xc004680d50, 0xc002f06840)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:268 +0x59
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc00042d518, 0xc004680d50, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:517 +0x20f
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00042d518, 0xc0009c6080, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377f20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 148 [semacquire]:
bytes.makeSlice(0x42c, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc000197138, 0xc2, 0x1)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc000197138, 0xc000cfb860, 0xc2, 0x5165d7, 0xc002db07ee, 0xc000904886)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000e49880)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000e49810)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000e490a0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc0019fb2f0, 0xaa03ab, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).isHighLinkDensity(0xc000197518, 0xc0019fb2f0, 0x2)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:566 +0x95
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc000197518, 0xc00039e440, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:386 +0x1ae
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000426000, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 149 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.ToLower(0xc0049ebdc0, 0xde, 0x7544a2, 0x0)
/usr/local/go/src/strings/strings.go:600 +0x222
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002be3588, 0xaa04f9, 0x2, 0xc0049ebdc0, 0xde, 0x30, 0xc002be3290, 0x42d308)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0x0, 0xc000c4ff80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:635 +0x82
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc000c4ff50, 0xc002be3320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc002be3518, 0xc004c3ae70, 0xc001247dc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc002be3518, 0xc004c3ae70, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc002be3518, 0xc004c3ae70, 0xc004c3ae70)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004260c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 151 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1dc1d9f80, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0038bfd50, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0038bfd40, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0010f4000, 0x438, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000a9e180, 0x32, 0xc0010f4000, 0x438, 0xc0010f4000, 0x438, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000a9e180, 0x32, 0xc0010f4000, 0x438, 0xaa0378, 0x1, 0xc0010f4000, 0x438, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000426240, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 153 [semacquire]:
github.com/PuerkitoBio/goquery.newSingleSelection(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:113
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc00419f770, 0xc0013ef320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0xb5
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc0013ef518, 0xc001793050, 0xc001001060)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc0013ef518, 0xc001793050, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc0013ef518, 0xc001793050, 0xc001793050)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004263c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 154 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).breakLongLines(0xc000ec30c0, 0xaa04bf, 0x2, 0xfc, 0x2, 0xc000ec2198)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:462 +0x58c
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc000ec30c0, 0xaa04bf, 0x2, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:426 +0x63
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc000ec30c0, 0xc000369f10, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:298 +0xb1
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000ec30c0, 0xc000369f10, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000ec30c0, 0xc000369f10, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc000ec30c0, 0xc001a167e0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000cc10c0, 0xc001a167e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000cc10c0, 0xc001a167e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc000ec30c0, 0xc001a16700, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000cc10c0, 0xc001a16700, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000cc10c0, 0xc001a16700, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc000ec30c0, 0xc0008660e0, 0xc001e94000, 0x7fe260fed560)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000cc10c0, 0xc0008660e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000cc10c0, 0xc0008660e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc000ec30c0, 0xc000866000, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000cc10c0, 0xc000866000, 0x54b170, 0xc0033a8000)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc000866000, 0x0, 0x0, 0x0, 0x0, 0xc000866000, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc001111e30, 0x0, 0x0, 0x0, 0x2000, 0xc0000fc000, 0x1c87, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0000fc000, 0x1c87, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000aa8540)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000cc1518, 0xc00103e080, 0xc001cfd520)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000426480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 155 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0022cb3c0, 0xaa0503, 0x2, 0xc002024600, 0x1978, 0x4, 0x4f3, 0xc002408760)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002024600, 0x1978, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0022cb518, 0xc001884f00, 0x56, 0xc00043eb40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000426540, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 156 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_Predict(0x7fe1f41bcff0, 0x7fe19934e910, 0x7fe19934ed50)
_cgo_gotypes.go:93 +0x4e
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.(*Model).Predict.func2(0xc00642bc00, 0x7fe19934e910, 0x7fe19934e910)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:144 +0x60
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.(*Model).Predict(0xc00642bc00, 0xc000968000, 0x340, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:144 +0xa8
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000968000, 0x340, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:31 +0xa0
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000f625a0, 0x4b, 0xc000968000, 0x340, 0xc000968000, 0x340, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000f625a0, 0x4b, 0xc000968000, 0x340, 0xaa0378, 0x1, 0xc000968000, 0x340, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000426600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 157 [semacquire]:
bytes.makeSlice(0x200, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc000e030e8, 0x200, 0xc004ed5300)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).Grow(...)
/usr/local/go/src/bytes/buffer.go:161
io/ioutil.readAll(0xb7b620, 0xc002d50750, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:34 +0xa3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc002d50750, 0xc000e031d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc002d50750, 0x0, 0x0, 0x0, 0x1300, 0xc004ed4000, 0x1012, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc004ed4000, 0x1012, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc000964240)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000e03518, 0xc00103fc40, 0xc00135ec40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004266c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 158 [semacquire]:
strings.genSplit(0xc006500000, 0x13b8, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc006164000, 0x3de, 0x3de)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00156b3c0, 0xaa059b, 0x2, 0xc0023ee000, 0x13b8, 0x1c, 0x3de, 0xc0032a9d60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0023ee000, 0x13b8, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00156b518, 0xc0031136e0, 0x42, 0xc0036501e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000426780, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 159 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc001fb0000, 0x14de, 0xc001817218, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc001fb0000, 0x14de, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0018173c0, 0xaa04d9, 0x2, 0xc001fb0000, 0x14de, 0x1, 0x48f, 0xc0034d1260)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001fb0000, 0x14de, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001817518, 0xc003112d40, 0x4f, 0xc0016a8af0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000426840, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 160 [semacquire]:
strings.genSplit(0xc004ce4000, 0x149d, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x149d, 0xc002372000, 0x14b9)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000785518, 0xc00039e620, 0xc0022265e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:321 +0x7b8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000426900, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 161 [semacquire]:
regexp.makeOnePass.func1(0x3, 0xc003351fa0, 0xf, 0xf, 0x1)
/usr/local/go/src/regexp/onepass.go:383 +0xb9b
regexp.makeOnePass(0xc003b37860, 0xc003b37860)
/usr/local/go/src/regexp/onepass.go:448 +0x398
regexp.compileOnePass(0xc003b37800, 0xc003b37800)
/usr/local/go/src/regexp/onepass.go:500 +0x129
regexp.compile(0xabc10c, 0x11, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc000ac3300, 0x0, 0xabc106, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc000ac3300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc000ac3300, 0x0, 0xb856e0, 0xc00039e048, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc000ac3300, 0xc000a16000, 0x0, 0x8, 0x434a07)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc000ac3300, 0x42ca96, 0xc003b377d0, 0x30, 0x28, 0xa4b3a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xabc101, 0x1d, 0xc000ac3370, 0x42d308, 0x30, 0xa4b3a0, 0xb85601)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xabc101, 0x1d, 0xc005a651e0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc001d08810, 0xabc101, 0x1d, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc000ac3518, 0xc00115a740, 0xabc101, 0x1d, 0x3e, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:133 +0x3a7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004269c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 162 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005bcfb60, 0xc0015652c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0015653c0, 0xaa0569, 0x2, 0xc001c9a000, 0x820, 0x1, 0x1e1, 0xc00137fa00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001c9a000, 0x820, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001565518, 0xc0012474c0, 0x41, 0xc001126a00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000426a80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 163 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc0009c6b00, 0xc000699220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000699588, 0xaa04f9, 0x2, 0xc0063e2c00, 0xfe, 0xd, 0x2a, 0xc0009c6ac0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc000699518, 0xc00115aec0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000426b40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 164 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc0042f9000, 0xea4, 0xc001713218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc0042f9000, 0xea4, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0017133c0, 0xaa0569, 0x2, 0xc0042f9000, 0xea4, 0x51, 0x2e9, 0xc004cdae00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0042f9000, 0xea4, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001713518, 0xc000c44400, 0x4b, 0xc000607d60)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000426c00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 165 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe144001210, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002270680, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002270670, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0036b5500, 0x6c1, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000f6269d, 0x42, 0xc0036b5500, 0x6c1, 0xc0036b5500, 0x6c1, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000f6269d, 0x42, 0xc0036b5500, 0x6c1, 0xaa0378, 0x1, 0xc0036b5500, 0x6c1, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000426cc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 166 [runnable]:
encoding/json.stateInString(0xc0015310a8, 0xa78f22, 0x0)
/usr/local/go/src/encoding/json/scanner.go:331 +0x5c
encoding/json.checkValid(0xc00424d440, 0x10c, 0x120, 0xc0015310a8, 0x120, 0x10c)
/usr/local/go/src/encoding/json/scanner.go:34 +0xa7
encoding/json.Unmarshal(0xc00424d440, 0x10c, 0x120, 0x9cfa40, 0xc00608dc00, 0x120, 0x1)
/usr/local/go/src/encoding/json/decode.go:101 +0x66
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.(*Model).Predict(0xc0000b3320, 0xc002fc7b00, 0x864, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:153 +0x16a
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLangFT(0xc002fc7b00, 0x864, 0xaa04f9, 0x2)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesft.go:14 +0x5f
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001126cd0, 0x3c, 0xc002fc7b00, 0x864, 0xc002fc7b00, 0x864, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:12 +0x6b
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001126cd0, 0x3c, 0xc002fc7b00, 0x864, 0xaa0378, 0x1, 0xc002fc7b00, 0x864, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000426d80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 169 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc0009c7ca0, 0xc0017cd2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0017cd3c0, 0xaa04f9, 0x2, 0xc000edaa00, 0x313c, 0x1, 0x97a, 0xc001000380)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000edaa00, 0x313c, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0017cd518, 0xc001247980, 0x49, 0xc00112678a)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000426fc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 170 [semacquire]:
bytes.makeSlice(0x81, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0022c23d8, 0x1, 0x1)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0022c23d8, 0x1189d52, 0x1, 0x1, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0022c23d8, 0xc0009f95e0, 0x9e, 0x9e, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:440 +0xf7
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0022c23d8, 0xc0017c1880, 0xc0054b62c0, 0xc0022c2308)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0022c23d8, 0xc0017c1810, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0022c30c0, 0xc0017c1810, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:155 +0x7cc
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0022c30c0, 0xc0017c1810, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0022c30c0, 0xc0017c15e0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0017f30c0, 0xc0017c15e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0017f30c0, 0xc0017c15e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0022c30c0, 0xc0017c1500, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0017f30c0, 0xc0017c1500, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0017f30c0, 0xc0017c1500, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0022c30c0, 0xc0017c0e00, 0xc000151c00, 0x7fe260fec108)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0017f30c0, 0xc0017c0e00, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0017f30c0, 0xc0017c0e00, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0022c30c0, 0xc0017c0d20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0017f30c0, 0xc0017c0d20, 0x54b170, 0xc006345110)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc0017c0d20, 0x0, 0x0, 0x0, 0x0, 0xc0017c0d20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc005a62810, 0x0, 0x0, 0x0, 0x1300, 0xc001661900, 0x11bd, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc001661900, 0x11bd, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc0031e1ec0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0017f3518, 0xc002409ea0, 0xc0000b3ce0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427080, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 171 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).breakLongLines(0xc0052b90c0, 0xaa04bf, 0x2, 0x11b, 0x2, 0xc0052b8198)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:462 +0x58c
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0052b90c0, 0xaa04bf, 0x2, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:426 +0x63
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0052b90c0, 0xc002d68ee0, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:298 +0xb1
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0052b90c0, 0xc002d68ee0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0052b90c0, 0xc002d68ee0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052b90c0, 0xc0019c8460, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000e190c0, 0xc0019c8460, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000e190c0, 0xc0019c8460, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052b90c0, 0xc0019c8380, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000e190c0, 0xc0019c8380, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000e190c0, 0xc0019c8380, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052b90c0, 0xc001c1dab0, 0xc000e18d40, 0x46a81e)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000e190c0, 0xc001c1dab0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000e190c0, 0xc001c1dab0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052b90c0, 0xc001c1d960, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000e190c0, 0xc001c1d960, 0x54b170, 0xc0036fc8f0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc001c1d960, 0x0, 0x0, 0x0, 0x0, 0xc001c1d960, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc001a422d0, 0x0, 0x0, 0x0, 0x1000, 0xc00440b000, 0xf5a, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc00440b000, 0xf5a, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc0005cde80)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000e19518, 0xc00039ff20, 0xc000b32f40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427140, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 172 [semacquire]:
strings.genSplit(0xc0056ba000, 0x98c, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xb5c, 0xc005471500, 0x98c)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000703588, 0xaa04f9, 0x2, 0xc005471500, 0x98c, 0x0, 0xc001e94000, 0x7fe260fee9b8)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc000703518, 0xc00135efc0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427200, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 173 [semacquire]:
github.com/PuerkitoBio/goquery.getChildrenWithSiblingType(0xc00157b420, 0x0, 0x0, 0x0, 0x75c780, 0xffffffffffffffff, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:667 +0x18f
github.com/PuerkitoBio/goquery.getChildrenNodes.func1(0x0, 0xc00157b420, 0xc006220001, 0xc0062201e0, 0xc0009f32d0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:607 +0x3c
github.com/PuerkitoBio/goquery.mapNodes(0xc0035cdac8, 0x1, 0x1, 0xc0009f3368, 0xc0009f3518, 0xc002d60bd0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:693 +0xf6
github.com/PuerkitoBio/goquery.getChildrenNodes(0xc0035cdac8, 0x1, 0x1, 0x0, 0xc001153560, 0xc0023a3f70, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:606 +0x6b
github.com/PuerkitoBio/goquery.(*Selection).Children(0xc002d60bd0, 0xc002d60bd0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:86 +0x4d
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc0009f3518, 0xc002d60bd0, 0xc002d60bd0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:719 +0x52
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004272c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 174 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005a65440, 0xc00236d220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00236d588, 0xaa04f9, 0x2, 0xc005c53400, 0xa91, 0xd, 0x22, 0xc005a65400)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00236d518, 0xc0013969c0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427380, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 175 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe170001080, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000cf87c0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc000cf87b0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002926b00, 0x52c, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000906240, 0x46, 0xc002926b00, 0x52c, 0xc002926b00, 0x52c, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000906240, 0x46, 0xc002926b00, 0x52c, 0xaa0378, 0x1, 0xc002926b00, 0x52c, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427440, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 176 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe194217670, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000dfbd90, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc000dfbd80, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00319ea80, 0x938, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000ea70e0, 0x43, 0xc00319ea80, 0x938, 0xc00319ea80, 0x938, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000ea70e0, 0x43, 0xc00319ea80, 0x938, 0xaa0378, 0x1, 0xc00319ea80, 0x938, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427500, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 177 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc0002361a0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:901 +0x820
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0002361a0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0002361a0, 0xc001be0070, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00229d520, 0x0, 0x0, 0x0, 0xc001169200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00229d520, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0011698f8, 0xc000b4aa00, 0x2255, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004275c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 178 [semacquire]:
bytes.makeSlice(0x40f, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0051650c0, 0x1, 0x1)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0051650c0, 0x1189d4c, 0x1, 0x1, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0051650c0, 0xc002db0fc0, 0x1b4, 0x7, 0xc002db0fc0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:440 +0xf7
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0051650c0, 0xc000568150, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:176 +0xb28
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0051650c0, 0xc000568150, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0051650c0, 0xc00351b030, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0017210c0, 0xc00351b030, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0017210c0, 0xc00351b030, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0051650c0, 0xc003884e00, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0017210c0, 0xc003884e00, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0017210c0, 0xc003884e00, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0051650c0, 0xc0010abab0, 0xc001e94000, 0x7fe260fee9b8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0017210c0, 0xc0010abab0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0017210c0, 0xc0010abab0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0051650c0, 0xc0010ab9d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0017210c0, 0xc0010ab9d0, 0x54b170, 0xc002d2a4e0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc0010ab9d0, 0x0, 0x0, 0x0, 0x0, 0xc0010ab9d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0030170b0, 0x0, 0x0, 0x0, 0x2000, 0xc001d0a000, 0x1dc3, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc001d0a000, 0x1dc3, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc0025fe000)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001721518, 0xc002488d60, 0xc002489860)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427680, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 179 [semacquire]:
regexp.(*Regexp).expand(0xc000149400, 0xc004624200, 0x40, 0x40, 0xaa0378, 0x1, 0x0, 0x0, 0x0, 0xc005b7c1e0, ...)
/usr/local/go/src/regexp/regexp.go:948 +0xa9b
regexp.(*Regexp).ReplaceAllString.func1(0xc004624200, 0x40, 0x40, 0xc003ce82c0, 0x2, 0x2, 0xc005b7c1e0, 0xe1, 0x3c)
/usr/local/go/src/regexp/regexp.go:558 +0xbf
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc005b7c1e0, 0xe1, 0x2, 0xc0014e0198, 0xc00000000a, 0x2, ...)
/usr/local/go/src/regexp/regexp.go:616 +0x2e9
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc005b7c1e0, 0xe1, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0014e10c0, 0xc00130c0e0, 0xc0014e01b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0014e10c0, 0xc00130c070, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0014e10c0, 0xc00130c070, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0014e10c0, 0xc00130c070, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0014e10c0, 0xc00130c070, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0014e10c0, 0xc00452a1c0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0014e10c0, 0xc00452a1c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0014e10c0, 0xc00452a1c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0014e10c0, 0xc00452a0e0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0014e10c0, 0xc00452a0e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0014e10c0, 0xc00452a0e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0014e10c0, 0xc005df9960, 0x7fe23810c3b0, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0014e10c0, 0xc005df9960, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0014e10c0, 0xc005df9960, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0014e10c0, 0xc005df9880, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0014e10c0, 0xc005df9880, 0x54b170, 0xc0039e28f0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc005df9880, 0x0, 0x0, 0x0, 0x0, 0xc005df9880, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc005cf4840, 0x0, 0x0, 0x0, 0x1000, 0xc005435000, 0xf23, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc005435000, 0xf23, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000767ac0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0014e1518, 0xc001397340, 0xc004c38700)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427740, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 180 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1ec0052a0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc003dc0b10, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc003dc0b00, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001892a00, 0x605, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000117180, 0x31, 0xc001892a00, 0x605, 0xc001892a00, 0x605, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000117180, 0x31, 0xc001892a00, 0x605, 0xaa0378, 0x1, 0xc001892a00, 0x605, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427800, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 181 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe138002c80, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0022405e0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0022405d0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000f1f000, 0x7d5, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00002b040, 0x3e, 0xc000f1f000, 0x7d5, 0xc000f1f000, 0x7d5, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00002b040, 0x3e, 0xc000f1f000, 0x7d5, 0xaa0378, 0x1, 0xc000f1f000, 0x7d5, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004278c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 182 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe234002260, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000fec500, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc000fec4f0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00084c700, 0x6c0, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000906a36, 0x46, 0xc00084c700, 0x6c0, 0xc00084c700, 0x6c0, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000906a36, 0x46, 0xc00084c700, 0x6c0, 0xaa0378, 0x1, 0xc00084c700, 0x6c0, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427980, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 183 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005bce120, 0xc002b83210, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002b83588, 0xaa04f9, 0x2, 0xc001f87600, 0x14c, 0x5, 0xc002b83290, 0x42d308)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0xd, 0xc0033cd8c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:635 +0x82
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc0041f0a50, 0xc002b83320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc002b83518, 0xc002d3c180, 0xc001246cc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc002b83518, 0xc002d3c180, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc002b83518, 0xc002d3c180, 0xc002d3c180)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427a40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 184 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc001885100, 0xc00187b2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00187b3c0, 0xaa04f9, 0x2, 0xc004b42000, 0x1f72, 0xc004b42000, 0xc00187b3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004b42000, 0x1f72, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00187b518, 0xc0013964e0, 0x42, 0xc000606460)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427b00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 186 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe118001220, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00090bb70, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00090bb60, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0037ef000, 0x163c, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0011266e0, 0x41, 0xc0037ef000, 0x163c, 0xc0037ef000, 0x163c, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0011266e0, 0x41, 0xc0037ef000, 0x163c, 0xaa0378, 0x1, 0xc0037ef000, 0x163c, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427c80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 187 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000e333c0, 0xaa04f9, 0x2, 0xc005040a80, 0x9d6, 0xc005040a80, 0xc000e333b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005040a80, 0x9d6, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000e33518, 0xc000b33ce0, 0x64, 0xc001116690)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427d40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 188 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc004b8c000, 0x10dc, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:71 +0x129
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc004b8c000, 0x10dc, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc004b8c000, 0x10dc, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc004b8c000, 0x10dc, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc004b8c000, 0x10dc, 0x0, 0x0, 0x10dc, 0xc004b8c000, 0x10dc)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc004b8c000, 0x10dc, 0x10dc, 0x1300)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000479e00, 0x60, 0xc004b8c000, 0x10dc, 0xc004b8c000, 0x10dc, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000479e00, 0x60, 0xc004b8c000, 0x10dc, 0xaa0378, 0x1, 0xc004b8c000, 0x10dc, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427e00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 189 [semacquire]:
github.com/jaytaylor/html2text.FromString(0xc00349c000, 0x10d0, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000aa8040)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:102 +0x42
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000583518, 0xc0000b2880, 0xc003213120)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427ec0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 190 [semacquire]:
strings.genSplit(0xc0063bb080, 0x550, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc002576000, 0x16b, 0x16b)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000b973c0, 0xaa0599, 0x2, 0xc000388580, 0x550, 0x1, 0x16b, 0xc001000e60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000388580, 0x550, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000b97518, 0xc000fef4e0, 0x37, 0xc000b73513)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427f80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 192 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2340a0950, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc003af6df0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc003af6de0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002b5ea80, 0x987, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0012bc210, 0x2a, 0xc002b5ea80, 0x987, 0xc002b5ea80, 0x987, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0012bc210, 0x2a, 0xc002b5ea80, 0x987, 0xaa0378, 0x1, 0xc002b5ea80, 0x987, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043e120, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 193 [semacquire]:
bytes.makeSlice(0x1e00, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0006d30e8, 0x200, 0x800)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).ReadFrom(0xc0006d30e8, 0xb7b620, 0xc003989410, 0x203000, 0x203000, 0x203000)
/usr/local/go/src/bytes/buffer.go:202 +0x48
io/ioutil.readAll(0xb7b620, 0xc003989410, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:36 +0xe3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc003989410, 0xc0006d31d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc003989410, 0x0, 0x0, 0x0, 0xd80, 0xc002213b00, 0xd4d, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc002213b00, 0xd4d, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc00322fec0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0006d3518, 0xc001396a00, 0xc004cdbb60)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043e1e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 194 [semacquire]:
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc000e112e0, 0x20300100000000, 0x7fe10134dfff, 0xc000e11218, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:724 +0x716
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc000e112e0, 0xc001fcd800, 0x5, 0xc000e11230, 0x46620f)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc000e112e0, 0xc000e112f8, 0x43b3b7, 0x1252790, 0xc000e11200, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xaa64ee, 0x5, 0x7fe2381755a8, 0xc000e11301, 0x10100000042ab7a, 0x0, 0xe7b)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xaa64ee, 0x5, 0x0, 0x0, 0xaa0301)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc005032b10, 0xaa64ee, 0x5, 0xfec)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).getTitleUnmodified(0xc000e11518, 0xc001001740, 0xe7b, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:57 +0x47
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:128 +0x282
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043e2a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 195 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc002754d00, 0x862, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:81 +0x208
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc002754d00, 0x862, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc002754d00, 0x862, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc002754d00, 0x862, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc002754d00, 0x862, 0x0, 0x0, 0x862, 0xc002754d00, 0x862)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc002754d00, 0x862, 0x862, 0x900)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000764320, 0x4f, 0xc002754d00, 0x862, 0xc002754d00, 0x862, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000764320, 0x4f, 0xc002754d00, 0x862, 0xaa0378, 0x1, 0xc002754d00, 0x862, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043e360, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 196 [runnable]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000e1d3c0, 0xaa04f9, 0x2, 0xc003d2d300, 0x108d, 0x1, 0x334, 0xc003309de0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003d2d300, 0x108d, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000e1d518, 0xc000b32480, 0x3c, 0xc003355a80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043e420, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 197 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).breakLongLines(0xc0013b30c0, 0xaa04bf, 0x2, 0x96, 0x2, 0xc0013b2198)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:462 +0x58c
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0013b30c0, 0xaa04bf, 0x2, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:426 +0x63
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0013b30c0, 0xc000f68ee0, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:298 +0xb1
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0013b30c0, 0xc000f68ee0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0013b30c0, 0xc000f68ee0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0013b30c0, 0xc000de9e30, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0007b10c0, 0xc000de9e30, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007b10c0, 0xc000de9e30, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0013b30c0, 0xc000de9d50, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0007b10c0, 0xc000de9d50, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007b10c0, 0xc000de9d50, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0013b30c0, 0xc000e802a0, 0xc001f60700, 0x7fe260fedc28)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0007b10c0, 0xc000e802a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007b10c0, 0xc000e802a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0013b30c0, 0xc000e801c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007b10c0, 0xc000e801c0, 0x54b170, 0xc000643ba0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc000e801c0, 0x0, 0x0, 0x0, 0x0, 0xc000e801c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc002e2ee70, 0x0, 0x0, 0x0, 0xc80, 0xc002e68000, 0xc80, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc002e68000, 0xc80, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000c50480)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0007b1518, 0xc00229c300, 0xc00135e700)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043e4e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 198 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe21c228940, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0028f1570, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0028f1560, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003dc6a80, 0xa51, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000478540, 0x4e, 0xc003dc6a80, 0xa51, 0xc003dc6a80, 0xa51, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000478540, 0x4e, 0xc003dc6a80, 0xa51, 0xaa0378, 0x1, 0xc003dc6a80, 0xa51, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043e5a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 199 [semacquire]:
github.com/gigawattio/window.Rolling(0xc005ea0000, 0x1c2, 0x1c2, 0x3, 0x0, 0x1c1, 0xc005ea0000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0007cd518, 0xc001397b00, 0xc0012465e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043e660, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 200 [semacquire]:
strings.genSplit(0xc0055c4000, 0x12a0, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x42f9b8, 0x38e, 0x38e)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0007c73c0, 0xaa059b, 0x2, 0xc00379a000, 0x12a0, 0x1, 0x38e, 0xc004c39ae0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00379a000, 0x12a0, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0007c7518, 0xc004c39700, 0x4f, 0xc000765d60)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043e720, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 201 [semacquire]:
bytes.makeSlice(0x1e00, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc000db30e8, 0x200, 0x800)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).ReadFrom(0xc000db30e8, 0xb7b620, 0xc005bb3590, 0x203001, 0x203001, 0x203001)
/usr/local/go/src/bytes/buffer.go:202 +0x48
io/ioutil.readAll(0xb7b620, 0xc005bb3590, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:36 +0xe3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc005bb3590, 0xc000db31d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc005bb3590, 0x0, 0x0, 0x0, 0x1000, 0xc005f36000, 0xea8, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc005f36000, 0xea8, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc002293a40)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000db3518, 0xc000feede0, 0xc0007630c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043e7e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 202 [semacquire]:
github.com/araddon/dateparse.newParser(...)
/home/develop/go/src/github.com/araddon/dateparse/parseany.go:1703
github.com/araddon/dateparse.parseTime(0xc00033f120, 0x12, 0x0, 0xc00033f120, 0x12, 0x12)
/home/develop/go/src/github.com/araddon/dateparse/parseany.go:214 +0x42
github.com/araddon/dateparse.ParseAny(0xc00033f120, 0x12, 0x3, 0xaa0378, 0x1, 0xc00033f120, 0x12)
/home/develop/go/src/github.com/araddon/dateparse/parseany.go:124 +0x42
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0007c9518, 0xc00039e2c0, 0xc0010015c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:344 +0xbce
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043e8a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 203 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2340a0490, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002ea3bb0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002ea3ba0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00399a000, 0x62b, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000ccc680, 0x24, 0xc00399a000, 0x62b, 0xc00399a000, 0x62b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000ccc680, 0x24, 0xc00399a000, 0x62b, 0xaa0378, 0x1, 0xc00399a000, 0x62b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043e960, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 204 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1981c6820, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc004097bf0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc004097be0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001271800, 0x584, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001272fc0, 0x39, 0xc001271800, 0x584, 0xc001271800, 0x584, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001272fc0, 0x39, 0xc001271800, 0x584, 0xaa0378, 0x1, 0xc001271800, 0x584, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043ea20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 205 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0016e8000, 0x85c, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:59 +0x21d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0016e8000, 0x85c, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0016e8000, 0x85c, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc0016e8000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0016e8000, 0x85c, 0x0, 0x0, 0x85c, 0x900, 0xc0016e8000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0016e8000, 0x85c, 0x85c, 0x900)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001418140, 0x49, 0xc0016e8000, 0x85c, 0xc0016e8000, 0x85c, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001418140, 0x49, 0xc0016e8000, 0x85c, 0xaa0378, 0x1, 0xc0016e8000, 0x85c, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043eae0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 206 [semacquire]:
strings.genSplit(0xc0034a2000, 0x11d3, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc006172000, 0x396, 0x396)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0008b93c0, 0xaa04fd, 0x2, 0xc005128c00, 0x11d3, 0x1, 0x396, 0xc005bcf6a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005128c00, 0x11d3, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0008b9518, 0xc001c5f000, 0x3a, 0xc000ccc640)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043eba0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 207 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc005848f00, 0x4b0, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:59 +0x21d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc005848f00, 0x4b0, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc005848f00, 0x4b0, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc005848f00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc005848f00, 0x4b0, 0x0, 0x0, 0x4b0, 0x580, 0xc005848f00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc005848f00, 0x4b0, 0x4b0, 0x580)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0012861c0, 0x3f, 0xc005848f00, 0x4b0, 0xc005848f00, 0x4b0, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0012861c0, 0x3f, 0xc005848f00, 0x4b0, 0xaa0378, 0x1, 0xc005848f00, 0x4b0, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043ec60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 208 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe19c44d130, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001b3c4c0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001b3c4a0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000a22e00, 0x685, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001127d10, 0x3c, 0xc000a22e00, 0x685, 0xc000a22e00, 0x685, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001127d10, 0x3c, 0xc000a22e00, 0x685, 0xaa0378, 0x1, 0xc000a22e00, 0x685, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043ed20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 209 [semacquire]:
bytes.makeSlice(0x1555, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc000c4e8a0, 0x35, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc000c4e8a0, 0xc001287680, 0x35, 0x6, 0xffffffffffffffff, 0x7fe260fec108)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc000c4e8a0, 0xc001287680, 0x35, 0xc002169290, 0x28)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc000c4e8a0, 0xc0022fad90, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc000c4e8a0, 0xc0022fad20, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc000c4e8a0, 0xc0022fac40, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc000c4e8a0, 0xc0022fab60, 0xb84d20, 0xc000c4e8a0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc000c4e8a0, 0xc0022fab60, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc000c4e870, 0x203000, 0x203000, 0x5d7039300000000, 0xc002611138)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs.func1(0x0, 0xc000c4e870)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:496 +0x67
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc000c4e840, 0xc0026112c8, 0x7)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs(0xc0026116b0, 0xc0018853e0, 0xaac03e, 0x7, 0xc0018853e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:495 +0xd8
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc0026116b0, 0xc0018853e0, 0xc001c5eb80)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:302 +0x39d
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043ede0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 210 [semacquire]:
regexp.(*Regexp).replaceAll(0xc0001fb400, 0x0, 0x0, 0x0, 0xc004972000, 0x140e, 0x2, 0xc00449fa58, 0xaa6444, 0x5, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc0001fb400, 0xc004972000, 0x140e, 0xaa0378, 0x1, 0xc005154000, 0x140e)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
_/home/ec2-user/environment/cluster2/cluster2/base.Sanitize(0xaa6444, 0x5, 0xc004972000, 0x140e, 0xaa0378, 0x1, 0xc004972000, 0x140e)
/home/ec2-user/environment/cluster2/cluster2/base/sanitizer.go:44 +0xc9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043eea0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:305 +0x37c
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 211 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1b02613b0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002f159a0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002f15990, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000fd9400, 0x4e1, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000aa8240, 0x36, 0xc000fd9400, 0x4e1, 0xc000fd9400, 0x4e1, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000aa8240, 0x36, 0xc000fd9400, 0x4e1, 0xaa0378, 0x1, 0xc000fd9400, 0x4e1, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043ef60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 212 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1c01a8a00, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00076e480, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00076e470, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000f8ea80, 0x356, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000376797, 0x41, 0xc000f8ea80, 0x356, 0xc000f8ea80, 0x356, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000376797, 0x41, 0xc000f8ea80, 0x356, 0xaa0378, 0x1, 0xc000f8ea80, 0x356, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f020, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 213 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1c41f9ac0, 0x7fe1c41f9c60)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0021ea080, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0021ea070, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001385500, 0x6f5, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000bba1e0, 0x3b, 0xc001385500, 0x6f5, 0xc001385500, 0x6f5, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000bba1e0, 0x3b, 0xc001385500, 0x6f5, 0xaa0378, 0x1, 0xc001385500, 0x6f5, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f0e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 214 [runnable]:
regexp.(*Regexp).doExecute(0xc000149400, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0024036c0, 0x3a, 0x0, 0x2, ...)
/usr/local/go/src/regexp/exec.go:521 +0x59b
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc0024036c0, 0x3a, 0x2, 0xc0068361d0, 0x0, 0xc006836270, ...)
/usr/local/go/src/regexp/regexp.go:599 +0x1a5
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc0024036c0, 0x3a, 0xaa0378, 0x1, 0xc005ab7f80, 0x7)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0068363d8, 0xc001ba1490, 0xc001aa13f0, 0xc006836308)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0068363d8, 0xc001ba1420, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0068370c0, 0xc001ba1420, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:155 +0x7cc
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0068370c0, 0xc001ba1420, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0068370c0, 0xc001ba1340, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00096b0c0, 0xc001ba1340, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00096b0c0, 0xc001ba1340, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0068370c0, 0xc001ba1260, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00096b0c0, 0xc001ba1260, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00096b0c0, 0xc001ba1260, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0068370c0, 0xc001ba0b60, 0xc001d7aa80, 0x7fe260fef080)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00096b0c0, 0xc001ba0b60, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00096b0c0, 0xc001ba0b60, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0068370c0, 0xc001ba0a80, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00096b0c0, 0xc001ba0a80, 0x54b170, 0xc0003871e0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc001ba0a80, 0x0, 0x0, 0x0, 0x0, 0xc001ba0a80, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc003c4b2c0, 0x0, 0x0, 0x0, 0xa80, 0xc003e4aa80, 0xa5f, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc003e4aa80, 0xa5f, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc00322ea80)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00096b518, 0xc001247300, 0xc0023007c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f1a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 215 [semacquire]:
github.com/fatih/set.newTS(...)
/home/develop/go/src/github.com/fatih/set/set_ts.go:15
github.com/fatih/set.New(...)
/home/develop/go/src/github.com/fatih/set/set.go:56
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00096d3c0, 0xaa0599, 0x2, 0xc003b6c000, 0x139c, 0xc003b6c000, 0xc00096d3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:74 +0x52
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003b6c000, 0x139c, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00096d518, 0xc0034d0020, 0x57, 0xc002717272)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f260, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 216 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc0009c6360, 0xc000c052c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000c053c0, 0xaa04f9, 0x2, 0xc00295aa00, 0x13e2, 0x4, 0x3fd, 0xc002226a00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00295aa00, 0x13e2, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000c05518, 0xc001247020, 0x4e, 0xc00002a6e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f320, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 217 [semacquire]:
strings.genSplit(0xc00066d900, 0xc76, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc002828000, 0x2ba, 0x2ba)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0008733c0, 0xaa0503, 0x2, 0xc0007ec000, 0xc76, 0x6, 0x2ba, 0xc001884400)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0007ec000, 0xc76, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000873518, 0xc000b32160, 0x5d, 0xc000479800)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f3e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 218 [semacquire]:
regexp.newQueue(...)
/usr/local/go/src/regexp/onepass.go:141
regexp.makeOnePass(0xc002c70150, 0xc002c70150)
/usr/local/go/src/regexp/onepass.go:306 +0x93
regexp.compileOnePass(0xc002c700f0, 0xc002c700f0)
/usr/local/go/src/regexp/onepass.go:500 +0x129
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0018e4ff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0018e4ff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc0013e2874)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0018e4ff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0018e4ff8, 0x1, 0xc005f2ac90, 0xc01009e5180, 0x280460fef080)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0018e4ff8, 0x4, 0x96a, 0xc003fe0601, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc003fe0000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc002873450, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc002c700c0, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0018e58f8, 0xc00072b800, 0x1636, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f4a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 219 [semacquire]:
regexp.compile(0xabc10c, 0x11, 0xd4, 0x10, 0xa2acc0, 0x10000)
/usr/local/go/src/regexp/regexp.go:186 +0x152
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc000e21300, 0x0, 0xabc106, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc000e21300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc000e21300, 0xc002722c00, 0x5, 0xc000e211d8, 0x46620f)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc000e21300, 0x1259780, 0x1259790, 0xc000e212a0, 0x43b3b7)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc000e21300, 0x42cb96, 0x0, 0x0, 0x0, 0xa4b3a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xabc101, 0x1d, 0xc000e21370, 0x42d308, 0x30, 0xa4b3a0, 0x1e65)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xabc101, 0x1d, 0xc0000b3f40, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc00135bc50, 0xabc101, 0x1d, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc000e21518, 0xc001246b60, 0xabc101, 0x1d, 0x21, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:133 +0x3a7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f560, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 220 [semacquire]:
github.com/fatih/set.newTS(...)
/home/develop/go/src/github.com/fatih/set/set_ts.go:15
github.com/fatih/set.New(...)
/home/develop/go/src/github.com/fatih/set/set.go:56
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0011b7400, 0xaa04f9, 0x2, 0xc001906000, 0xe8, 0xc0011b7240, 0x42d308, 0x8)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:74 +0x52
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords.func1(0x2, 0xc00291d0e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:174 +0x86
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc00291cff0, 0xc0011b72c8, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords(0xc0011b7388)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:173 +0xb0
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc0011b7388, 0xc000ff5830, 0xaa04f9, 0x2, 0x42d308, 0x8, 0x9ecc80, 0xc0011b7301, 0xc000900de0)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:43 +0xf5
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc0011b7518, 0xc000ff5830, 0xaa04f9, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f620, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 221 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe13c005dd0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001be6640, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001be6620, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00049e900, 0x883, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00087a0c0, 0x34, 0xc00049e900, 0x883, 0xc00049e900, 0x883, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00087a0c0, 0x34, 0xc00049e900, 0x883, 0xaa0378, 0x1, 0xc00049e900, 0x883, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f6e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 222 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc0002372b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc000237380, 0xc0039afab0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc001000400, 0x0, 0x0, 0x0, 0xc00131d200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc001000400, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00131d8f8, 0xc000fa1000, 0xe1d, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f7a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 223 [semacquire]:
bytes.makeSlice(0xf5b, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0004ad388, 0x8d, 0x4b)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0004ad388, 0xc0035eaa20, 0x8d, 0x5165d7, 0xc002bc9eef, 0xc000dfab68)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc003b399d0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc003b39960)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc003b38380)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc003b382a0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc003b35b90)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc001b62f90, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0004ad518, 0xc000763e00, 0x43, 0xc0016a9af1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f860, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 224 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc005c98000, 0xdd8, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:71 +0x129
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc005c98000, 0xdd8, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc005c98000, 0xdd8, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc005c98000, 0xdd8, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc005c98000, 0xdd8, 0x0, 0x0, 0xdd8, 0xc005c98000, 0xdd8)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc005c98000, 0xdd8, 0xdd8, 0x1300)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0006f9819, 0x60, 0xc005c98000, 0xdd8, 0xc005c98000, 0xdd8, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0006f9819, 0x60, 0xc005c98000, 0xdd8, 0xaa0378, 0x1, 0xc005c98000, 0xdd8, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f920, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 225 [semacquire]:
golang.org/x/net/html.NewTokenizerFragment(0xb7b620, 0xc002f06090, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1215 +0x43
golang.org/x/net/html.NewTokenizer(...)
/home/develop/go/src/golang.org/x/net/html/token.go:1201
golang.org/x/net/html.ParseWithOptions(0xb7b620, 0xc002f06090, 0x0, 0x0, 0x0, 0x0, 0x1a01, 0xc0011aad20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2337 +0x41
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0011aad20, 0x0, 0x0, 0x0, 0x1a80, 0xc004e38f80, 0x1a50, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:93 +0x76
github.com/jaytaylor/html2text.FromString(0xc004e38f80, 0x1a50, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000a9eec0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000eb9518, 0xc000762c60, 0xc000b33340)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f9e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 226 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2181c5e30, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00090bbe0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00090bbd0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000370300, 0x2f9, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000c5e380, 0x58, 0xc000370300, 0x2f9, 0xc000370300, 0x2f9, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000c5e380, 0x58, 0xc000370300, 0x2f9, 0xaa0378, 0x1, 0xc000370300, 0x2f9, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043faa0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 228 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1981c5cd0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0021df560, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0021df550, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0008ec380, 0xd77, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002eea050, 0x47, 0xc0008ec380, 0xd77, 0xc0008ec380, 0xd77, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002eea050, 0x47, 0xc0008ec380, 0xd77, 0xaa0378, 0x1, 0xc0008ec380, 0xd77, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043fc20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 229 [semacquire]:
github.com/PuerkitoBio/goquery.getParentNodes.func1(0x0, 0xc000b6bea0, 0xc000037800, 0xc000ee5140, 0x4341f3)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:680 +0x5f
github.com/PuerkitoBio/goquery.mapNodes(0xc000901470, 0x1, 0x1, 0xae5578, 0xc001999aa0, 0xc00103ef48, 0xae6208)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:693 +0xf6
github.com/PuerkitoBio/goquery.getParentNodes(0xc000901470, 0x1, 0x1, 0x1, 0x0, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:678 +0x4b
github.com/PuerkitoBio/goquery.(*Selection).Parent(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:106
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc000ee5518, 0xc000762060, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:432 +0x56d
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043fce0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 230 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe200239ae0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc004050220, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc004050200, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000a4ed00, 0x8fc, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000b5fb20, 0x69, 0xc000a4ed00, 0x8fc, 0xc000a4ed00, 0x8fc, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000b5fb20, 0x69, 0xc000a4ed00, 0x8fc, 0xaa0378, 0x1, 0xc000a4ed00, 0x8fc, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043fda0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 231 [semacquire]:
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc000aada40, 0xe1, 0x2, 0xc0011a21d0, 0xa, 0xc0011a2270, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc000aada40, 0xe1, 0xaa0378, 0x1, 0xc005ab7e00, 0x4)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0011a23d8, 0xc000d8bce0, 0xc001aa0000, 0xc0011a2308)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0011a23d8, 0xc000d8bc70, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0011a30c0, 0xc000d8bc70, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:155 +0x7cc
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0011a30c0, 0xc000d8bc70, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0011a30c0, 0xc000d8ba40, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0011a30c0, 0xc000d8ba40, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0011a30c0, 0xc000d8ba40, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0011a30c0, 0xc000d8b960, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0011a30c0, 0xc000d8b960, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0011a30c0, 0xc000d8b960, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0011a30c0, 0xc000d8b260, 0xc0011a2d40, 0x46a81e)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0011a30c0, 0xc000d8b260, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0011a30c0, 0xc000d8b260, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0011a30c0, 0xc000d8b180, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0011a30c0, 0xc000d8b180, 0x54b170, 0xc0033a9040)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc000d8b180, 0x0, 0x0, 0x0, 0x0, 0xc000d8b180, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc003380e70, 0x0, 0x0, 0x0, 0x1800, 0xc000fc2800, 0x16dc, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc000fc2800, 0x16dc, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc001122880)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0011a3518, 0xc0000b32c0, 0xc0033826e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043fe60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 232 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe19420c300, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000dff480, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc000dff460, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0036dad00, 0x8fb, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00007ee40, 0x5b, 0xc0036dad00, 0x8fb, 0xc0036dad00, 0x8fb, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00007ee40, 0x5b, 0xc0036dad00, 0x8fb, 0xaa0378, 0x1, 0xc0036dad00, 0x8fb, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043ff20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 233 [semacquire]:
regexp.(*Regexp).replaceAll(0xc000149540, 0x0, 0x0, 0x0, 0xc002534800, 0x157d, 0x2, 0xc001465038, 0xc002534800, 0x157d, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc000149540, 0xc002534800, 0x157d, 0xaa04bf, 0x2, 0x1, 0x2)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.FromHTMLNode(0xc0017c1730, 0x0, 0x0, 0x0, 0x0, 0xc0017c1730, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:80 +0x191
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc001069b30, 0x0, 0x0, 0x0, 0x1800, 0xc001887800, 0x17cc, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc001887800, 0x17cc, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc00087ae40)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000e07518, 0xc00039fa40, 0xc00229d080)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e000, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 235 [semacquire]:
bytes.makeSlice(0xf6, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00142d208, 0x76, 0x1a)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00142d208, 0xc000b64580, 0x76, 0x1a, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00019bc00)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00019ba40)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc0042e7740, 0xa9ece0, 0xc0042e7710)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00142d518, 0xc0010b0040, 0x44)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:384 +0x144
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e180, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 236 [semacquire]:
github.com/andybalholm/cascadia.Compile(0xaa03ab, 0x1, 0xc00039e088, 0xc00038e0f0, 0xc0037d3158)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:109 +0x76
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc001ca2690, 0xaa03ab, 0x1, 0xc004c39fe0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).isHighLinkDensity(0xc0037d3518, 0xc001ca2690, 0x2)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:562 +0x55
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0037d3518, 0xc00103fa20, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:386 +0x1ae
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e240, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 237 [runnable]:
strings.Index(0xc0040d4600, 0x1879, 0xaa0378, 0x1, 0x38d)
/usr/local/go/src/strings/strings.go:1028 +0x67a
strings.genSplit(0xc0040d4600, 0x1879, 0xaa0378, 0x1, 0x0, 0x38d, 0x1879, 0xc003d99980, 0x1891)
/usr/local/go/src/strings/strings.go:260 +0xf3
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000687518, 0xc0010b17c0, 0xc00137fd40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:321 +0x7b8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 238 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe110001320, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0010bf310, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0010bf2f0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003311200, 0x598, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00068c1e0, 0x5b, 0xc003311200, 0x598, 0xc003311200, 0x598, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00068c1e0, 0x5b, 0xc003311200, 0x598, 0xaa0378, 0x1, 0xc003311200, 0x598, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e3c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 239 [semacquire]:
strings.genSplit(0xc004547b00, 0xd21, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc00390d500, 0x297, 0x297)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0005753c0, 0xaa0599, 0x2, 0xc000868d80, 0xd21, 0x1, 0x297, 0xc005c54920)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000868d80, 0xd21, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000575518, 0xc00229d9e0, 0x39, 0xc001272540)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 240 [semacquire]:
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc0053ec500, 0x129, 0x2, 0xc005eea198, 0xc00000000a, 0x2, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc0053ec500, 0x129, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc005eeb0c0, 0xc004e16af0, 0xc005eea1b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005eeb0c0, 0xc004e16a80, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc005eeb0c0, 0xc004e16a80, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc005eeb0c0, 0xc004e16a80, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc005eeb0c0, 0xc004e16a80, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005eeb0c0, 0xc004e161c0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0005770c0, 0xc004e161c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0005770c0, 0xc004e161c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005eeb0c0, 0xc004e160e0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0005770c0, 0xc004e160e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0005770c0, 0xc004e160e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005eeb0c0, 0xc004ccb9d0, 0xc000576d40, 0x46a81e)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0005770c0, 0xc004ccb9d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0005770c0, 0xc004ccb9d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005eeb0c0, 0xc004ccb8f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0005770c0, 0xc004ccb8f0, 0x54b170, 0xc0006e9450)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc004ccb8f0, 0x0, 0x0, 0x0, 0x0, 0xc004ccb8f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc000ada2d0, 0x0, 0x0, 0x0, 0x1300, 0xc003ecb900, 0x12e2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc003ecb900, 0x12e2, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc0005cc140)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000577518, 0xc001c5e8a0, 0xc004cdb4a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e540, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 241 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0007813c0, 0xaa04f9, 0x2, 0xc000862000, 0x1380, 0x1, 0x413, 0xc001000660)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000862000, 0x1380, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000781518, 0xc000b33320, 0x53, 0xc000906d8e)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 242 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.(*Model).Predict(0xc0000b3320, 0xc00231aa80, 0x332, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:153 +0x132
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLangFT(0xc00231aa80, 0x332, 0xaa04f9, 0x2)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesft.go:14 +0x5f
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000117980, 0x31, 0xc00231aa80, 0x332, 0xc00231aa80, 0x332, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:12 +0x6b
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000117980, 0x31, 0xc00231aa80, 0x332, 0xaa0378, 0x1, 0xc00231aa80, 0x332, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e6c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 243 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc00308bc00, 0x6d7, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc00308bc00, 0x6d7, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc00308bc00, 0x6d7, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc00308bc00, 0x6d7, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00308bc00, 0x6d7, 0x0, 0x0, 0x6d7, 0xc00308bc00, 0x6d7)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00308bc00, 0x6d7, 0x6d7, 0x800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0011265f0, 0x49, 0xc00308bc00, 0x6d7, 0xc00308bc00, 0x6d7, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0011265f0, 0x49, 0xc00308bc00, 0x6d7, 0xaa0378, 0x1, 0xc00308bc00, 0x6d7, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e780, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 244 [semacquire]:
regexp/syntax.(*parser).maybeConcat(0xc002c0cc00, 0xffffffff, 0xc001366d60)
/usr/local/go/src/regexp/syntax/parse.go:165 +0x239
regexp/syntax.(*parser).push(0xc002c0cc00, 0xc003ba7ce0, 0xc003ba7ce0)
/usr/local/go/src/regexp/syntax/parse.go:136 +0x218
regexp/syntax.(*parser).literal(0xc002c0cc00, 0x6f)
/usr/local/go/src/regexp/syntax/parse.go:210 +0x53
regexp/syntax.Parse(0xabc10c, 0x11, 0xd4, 0x0, 0x0, 0xabc106)
/usr/local/go/src/regexp/syntax/parse.go:729 +0x2ae
regexp.compile(0xabc10c, 0x11, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:170 +0x5a
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc001367300, 0x0, 0xabc106, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc001367300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc001367300, 0x0, 0xb856e0, 0xc00039e1c8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc001367300, 0xc001401810, 0x0, 0x8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc001367300, 0x42ca96, 0xc001a6e030, 0x30, 0x28, 0xa4b3a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xabc101, 0x1d, 0xc001367370, 0x42d308, 0x30, 0xa4b3a0, 0xb85601)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xabc101, 0x1d, 0xc00608c660, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc001230b70, 0xabc101, 0x1d, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc001367518, 0xc00115a660, 0xabc101, 0x1d, 0x3d, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:133 +0x3a7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e840, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 246 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2041a8d30, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00284ea80, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00284ea70, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003ac5c00, 0x302, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0012bc3c7, 0x27, 0xc003ac5c00, 0x302, 0xc003ac5c00, 0x302, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0012bc3c7, 0x27, 0xc003ac5c00, 0x302, 0xaa0378, 0x1, 0xc003ac5c00, 0x302, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e9c0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 248 [semacquire]:
regexp.(*Regexp).replaceAll(0xc0001fb220, 0x0, 0x0, 0x0, 0xc005950000, 0xeb7, 0x2, 0xc0018ed218, 0xc0000a2350, 0x7de1b0, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc0001fb220, 0xc005950000, 0xeb7, 0xaa0378, 0x1, 0xc0009c6200, 0xc0038e6300)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/advancedlogic/GoOse.(*outputFormatter).getOutputText(0xc0018ed388, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:121 +0x81
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc0018ed388, 0xc001282630, 0xaa04f9, 0x2, 0x42d308, 0x8, 0x9ecc80, 0xc0018ed301, 0xc000011990)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:45 +0x103
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc0018ed518, 0xc001282630, 0xaa04f9, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045eb40, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 249 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc002d01560, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001101518, 0xc0000b3b40, 0x51, 0xc001fb63d6)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045ec00, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 251 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1ec001030, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0012181d0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0012181c0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0010ee000, 0x3c1, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000ccc947, 0x38, 0xc0010ee000, 0x3c1, 0xc0010ee000, 0x3c1, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000ccc947, 0x38, 0xc0010ee000, 0x3c1, 0xaa0378, 0x1, 0xc0010ee000, 0x3c1, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045ed80, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 252 [semacquire]:
regexp.(*Regexp).replaceAll(0xc000149ae0, 0x0, 0x0, 0x0, 0xc0022f4000, 0xfb7, 0x2, 0xc0017cfa58, 0xaa3a2a, 0x4, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc000149ae0, 0xc0022f4000, 0xfb7, 0x0, 0x0, 0xc00038e060, 0x127d7a0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
_/home/ec2-user/environment/cluster2/cluster2/base.Sanitize(0xaa3a2a, 0x4, 0xc0022f4000, 0xfb7, 0x0, 0x0, 0x10, 0xab52f4)
/home/ec2-user/environment/cluster2/cluster2/base/sanitizer.go:44 +0xc9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045ee40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:304 +0x33a
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 253 [semacquire]:
github.com/PuerkitoBio/goquery.appendWithoutDuplicates(0xc005e0e080, 0x9, 0x10, 0xc000a99b00, 0x14, 0x20, 0xc000a61298, 0xc00035de90, 0x1, 0xffffffffffffffff)
/home/develop/go/src/github.com/PuerkitoBio/goquery/utilities.go:138 +0x172
github.com/PuerkitoBio/goquery.mapNodes(0xc000a81f98, 0x1, 0x1, 0xc000a61368, 0xc000a61518, 0xc00160b290, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:694 +0x159
github.com/PuerkitoBio/goquery.getChildrenNodes(0xc000a81f98, 0x1, 0x1, 0x0, 0xc001b63380, 0xc0034189d0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:606 +0x6b
github.com/PuerkitoBio/goquery.(*Selection).Children(0xc00160b290, 0xc00160b290)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:86 +0x4d
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc000a61518, 0xc00160b290, 0xc00160b290)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:719 +0x52
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045ef00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 254 [runnable]:
bytes.makeSlice.func1()
/usr/local/go/src/bytes/buffer.go:224 +0x68
bytes.makeSlice(0x1e00, 0xc00695a000, 0x1e00, 0x1e00)
/usr/local/go/src/bytes/buffer.go:229 +0x96
bytes.(*Buffer).grow(0xc0008b30e8, 0x200, 0x800)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).ReadFrom(0xc0008b30e8, 0xb7b620, 0xc005cf5bf0, 0x203001, 0x203001, 0x203001)
/usr/local/go/src/bytes/buffer.go:202 +0x48
io/ioutil.readAll(0xb7b620, 0xc005cf5bf0, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:36 +0xe3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc005cf5bf0, 0xc0008b31d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc005cf5bf0, 0x0, 0x0, 0x0, 0xd80, 0xc0013ddb00, 0xcb8, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc0013ddb00, 0xcb8, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc0022f2b40)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0008b3518, 0xc0010b1800, 0xc000feffc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045efc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 255 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe154004d50, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0033c0c80, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0033c0c70, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001f67c00, 0x3b1, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000e84cc0, 0x3e, 0xc001f67c00, 0x3b1, 0xc001f67c00, 0x3b1, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000e84cc0, 0x3e, 0xc001f67c00, 0x3b1, 0xaa0378, 0x1, 0xc001f67c00, 0x3b1, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045f080, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 256 [semacquire]:
github.com/gigawattio/window.Rolling(0xc0012c8000, 0x132, 0x132, 0x3, 0x0, 0x131, 0xc0012c8000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001039518, 0xc000c44500, 0xc000c44e80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045f140, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 257 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0008b53c0, 0xaa04f9, 0x2, 0xc002e26000, 0xe6f, 0x1, 0x309, 0xc000762440)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002e26000, 0xe6f, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0008b5518, 0xc0034d0d00, 0x63, 0xc000dea0e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045f200, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 258 [runnable]:
github.com/fatih/set.(*Set).Add(0xc0032a86e0, 0xc000fe52c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:26 +0x124
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000fe53c0, 0xaa04f9, 0x2, 0xc004164a00, 0x147d, 0xc004164a00, 0xc000fe53b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004164a00, 0x147d, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000fe5518, 0xc001c5f220, 0x4e, 0xc000607c70)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045f2c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 259 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0013b70c0, 0xc005b7cf00, 0xe4, 0xe4, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:430 +0x233
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0013b70c0, 0xc003a1b260, 0xc0013b61b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0013b70c0, 0xc003a1b1f0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0013b70c0, 0xc003a1b1f0, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0013b70c0, 0xc003a1b1f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0013b70c0, 0xc003a1b1f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0013b70c0, 0xc001608e70, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00116b0c0, 0xc001608e70, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00116b0c0, 0xc001608e70, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0013b70c0, 0xc001608d90, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00116b0c0, 0xc001608d90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00116b0c0, 0xc001608d90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0013b70c0, 0xc001608690, 0xc0002f9c00, 0x7fe260fed560)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00116b0c0, 0xc001608690, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00116b0c0, 0xc001608690, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0013b70c0, 0xc0016085b0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00116b0c0, 0xc0016085b0, 0x54b170, 0xc000899110)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc0016085b0, 0x0, 0x0, 0x0, 0x0, 0xc0016085b0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc000cfeea0, 0x0, 0x0, 0x0, 0x2800, 0xc002152800, 0x2628, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc002152800, 0x2628, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xffffffffffffffff, 0x3f00006b00000004, 0xc0025028c0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00116b518, 0xc002226460, 0xc002226d00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045f380, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 260 [runnable]:
reflect.Value.Type(0x9cfa40, 0xc00642b6c0, 0x16, 0x9de620, 0xc00642b6c0)
/usr/local/go/src/reflect/value.go:1869 +0x189
encoding/json.indirect(0x9cfa40, 0xc00642b6c0, 0x16, 0x0, 0x100000000000000, 0x0, 0x10, 0xc002271e60, 0x1259790, 0x10000c0041317b0, ...)
/usr/local/go/src/encoding/json/decode.go:491 +0x1d8
encoding/json.(*decodeState).array(0xc0052213f0, 0x9cfa40, 0xc00642b6c0, 0x16, 0xc005221418, 0x5b)
/usr/local/go/src/encoding/json/decode.go:516 +0x5f
encoding/json.(*decodeState).value(0xc0052213f0, 0x9cfa40, 0xc00642b6c0, 0x16, 0xc0041318a8, 0x532927)
/usr/local/go/src/encoding/json/decode.go:377 +0xfd
encoding/json.(*decodeState).unmarshal(0xc0052213f0, 0x9cfa40, 0xc00642b6c0, 0xc005221418, 0x0)
/usr/local/go/src/encoding/json/decode.go:180 +0x1f0
encoding/json.Unmarshal(0xc002d2ec80, 0x124, 0x140, 0x9cfa40, 0xc00642b6c0, 0x140, 0xaa366e)
/usr/local/go/src/encoding/json/decode.go:107 +0x112
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.(*Model).Predict(0xc00642b6a0, 0xc002f1e800, 0x78a, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:153 +0x16a
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002f1e800, 0x78a, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:31 +0xa0
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000a9e000, 0x37, 0xc002f1e800, 0x78a, 0xc002f1e800, 0x78a, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000a9e000, 0x37, 0xc002f1e800, 0x78a, 0xaa0378, 0x1, 0xc002f1e800, 0x78a, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045f440, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 261 [semacquire]:
strings.genSplit(0xc00330a000, 0x58c, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc00520b900, 0x11a, 0x11a)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00116d3c0, 0xaa0599, 0x2, 0xc00195b200, 0x58c, 0x1, 0x11a, 0xc0007639c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00195b200, 0x58c, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00116d518, 0xc00115b200, 0x2f, 0xc0012bc1e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045f500, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 262 [semacquire]:
github.com/gigawattio/window.Rolling(0xc005e52000, 0x100, 0x100, 0x3, 0x0, 0xff, 0xc005e52000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0017cf518, 0xc0009c74c0, 0xc000c44060)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045f5c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 413 [semacquire]:
bytes.makeSlice(0x139f, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00245fa40, 0x139f, 0x5b)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).Grow(...)
/usr/local/go/src/bytes/buffer.go:161
io/ioutil.readAll(0xb7c120, 0xc000383a50, 0x139f, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:34 +0xa3
io/ioutil.ReadFile(0xc000897bc0, 0x5b, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:73 +0xed
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000897bc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:286 +0x8a
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 264 [semacquire]:
strings.genSplit(0xc00177a000, 0x5f1, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xb8a, 0xc005f39800, 0x5f1)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002f9d588, 0xaa04f9, 0x2, 0xc005f39800, 0x5f1, 0x1, 0x12e, 0xc00656dbe0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc002f9d518, 0xc0012476a0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045f740, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 265 [semacquire]:
bytes.makeSlice(0x1d1b, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc001e4f128, 0x105, 0x2)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc001e4f128, 0xc000964a20, 0x105, 0x2, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc001e40b60)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc001e3ed20)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc001e3e700)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc00040ccf0, 0xaa03ab, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).isHighLinkDensity(0xc001e4f518, 0xc00040ccf0, 0x2)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:566 +0x95
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0x0, 0xc00040ccf0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:636 +0xa4
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc00040ccc0, 0xc001e4f320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc001e4f518, 0xc00040cc30, 0xc005ca7100)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc001e4f518, 0xc00040cc30, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc001e4f518, 0xc00040cc30, 0xc00040cc30)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045f800, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 266 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.ToLower(0xc0017b9500, 0x649, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:600 +0x222
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0006833c0, 0xaa0503, 0x2, 0xc0017b9500, 0x649, 0x5, 0x171, 0xc0009c6900)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0017b9500, 0x649, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000683518, 0xc000fef760, 0x43, 0xc001127c82)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045f8c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 267 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2200014d0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0015fdc60, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0015fdc40, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003365800, 0x5ed, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0012bc3a1, 0x1b, 0xc003365800, 0x5ed, 0xc003365800, 0x5ed, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0012bc3a1, 0x1b, 0xc003365800, 0x5ed, 0xaa0378, 0x1, 0xc003365800, 0x5ed, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045f980, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 269 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2340053e0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002fdf130, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002fdf120, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001620a00, 0x4ff, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001286cc0, 0x34, 0xc001620a00, 0x4ff, 0xc001620a00, 0x4ff, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001286cc0, 0x34, 0xc001620a00, 0x4ff, 0xaa0378, 0x1, 0xc001620a00, 0x4ff, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045fb00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 270 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc004cdb420, 0xc000cab2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000cab3c0, 0xaa0569, 0x2, 0xc000da6000, 0x89c, 0x2f, 0x1b7, 0xc002488020)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000da6000, 0x89c, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000cab518, 0xc00135f8c0, 0x37, 0xc001273e00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045fbc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 271 [runnable]:
bytes.NewReader(...)
/usr/local/go/src/bytes/reader.go:160
github.com/jaytaylor/html2text.FromString(0xc002c1ac00, 0xbfd, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc00038dfb0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0xb0
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000871518, 0xc00135fac0, 0xc00642b600)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045fc80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 272 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe234002520, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0027e99e0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0027e99d0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002384000, 0x742, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00002af50, 0x48, 0xc002384000, 0x742, 0xc002384000, 0x742, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00002af50, 0x48, 0xc002384000, 0x742, 0xaa0378, 0x1, 0xc002384000, 0x742, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045fd40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 273 [semacquire]:
strings.genSplit(0xc0035a2a80, 0x980, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc00546c000, 0x1ed, 0x1ed)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0014dd3c0, 0xaa0503, 0x2, 0xc00124b500, 0x980, 0x4, 0x1ed, 0xc00229cd60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00124b500, 0x980, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0014dd518, 0xc0000b2a20, 0x33, 0xc000aa8cc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045fe00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 274 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe230261710, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002d7b860, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002d7b850, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001345700, 0x527, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000a9e340, 0x3b, 0xc001345700, 0x527, 0xc001345700, 0x527, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000a9e340, 0x3b, 0xc001345700, 0x527, 0xaa0378, 0x1, 0xc001345700, 0x527, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045fec0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 275 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc002294000, 0xede, 0x1300, 0xc002294000)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc002294000, 0xede, 0xf1f, 0x0)
/usr/local/go/src/strings/strings.go:610 +0x8f
_/home/ec2-user/environment/cluster2/cluster2/base.Sanitize(0xaa3a2a, 0x4, 0xc002245000, 0xf1f, 0x0, 0x0, 0x10, 0xab52f4)
/home/ec2-user/environment/cluster2/cluster2/base/sanitizer.go:44 +0xe1
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045ff80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:304 +0x33a
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 276 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc00190e800, 0x721, 0xc000ca9218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc00190e800, 0x721, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000ca93c0, 0xaa05c9, 0x2, 0xc00190e800, 0x721, 0x3, 0x1ca, 0xc0000b3fa0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00190e800, 0x721, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000ca9518, 0xc000fee640, 0x37, 0xc001273b40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000478060, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 277 [semacquire]:
github.com/andybalholm/cascadia.Compile(0xaa03ab, 0x1, 0x1, 0xb856e0, 0xc001397fe0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:109 +0x76
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc001290270, 0xaa03ab, 0x1, 0xc000c4fb90)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetTags(0xc000e31518, 0xc00103fd80, 0xc0014f0f80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:268 +0xdf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:140 +0x505
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000478120, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 278 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._cgo_cmalloc(0x12f0, 0x7fe201965e30)
_cgo_gotypes.go:138 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_CString(0xc00165cc00, 0x12ef, 0xc000234750)
_cgo_gotypes.go:47 +0x33
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.(*Model).Predict(0xc0000b3320, 0xc00165cc00, 0x12ef, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:139 +0x6b
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLangFT(0xc00165cc00, 0x12ef, 0xaa04f9, 0x2)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesft.go:14 +0x5f
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000b5fa40, 0x65, 0xc00165cc00, 0x12ef, 0xc00165cc00, 0x12ef, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:12 +0x6b
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000b5fa40, 0x65, 0xc00165cc00, 0x12ef, 0xaa0378, 0x1, 0xc00165cc00, 0x12ef, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004781e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 279 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc0022266a0, 0xc001751220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001751588, 0xaa04f9, 0x2, 0xc000c63130, 0xb0, 0x7, 0x1b, 0xc002226660)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc001751518, 0xc000b325e0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004782a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 280 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0014f8a00, 0x634, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:81 +0x208
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0014f8a00, 0x634, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0014f8a00, 0x634, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0014f8a00, 0x634, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0014f8a00, 0x634, 0x0, 0x0, 0x634, 0xc0014f8a00, 0x634)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0014f8a00, 0x634, 0x634, 0x700)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000aa9f80, 0x3e, 0xc0014f8a00, 0x634, 0xc0014f8a00, 0x634, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000aa9f80, 0x3e, 0xc0014f8a00, 0x634, 0xaa0378, 0x1, 0xc0014f8a00, 0x634, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000478360, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 281 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1400016e0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001552920, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001552910, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002e4fc00, 0xb93, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00007ff80, 0x53, 0xc002e4fc00, 0xb93, 0xc002e4fc00, 0xb93, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00007ff80, 0x53, 0xc002e4fc00, 0xb93, 0xaa0378, 0x1, 0xc002e4fc00, 0xb93, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000478420, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 282 [semacquire]:
github.com/gigawattio/window.Rolling(0xc00486d980, 0x192, 0x192, 0x3, 0x0, 0x191, 0xc00486d980)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:10 +0x6c
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0010e1518, 0xc0034d0dc0, 0xc0024081a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004784e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 283 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0012ba030, 0x50d201, 0xc0009060c0, 0x57, 0x3f)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000f23518, 0xc001397520, 0xc000b32980)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:285 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004785a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 284 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc00315c000, 0x7ef, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:59 +0x21d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc00315c000, 0x7ef, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc00315c000, 0x7ef, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00315c000, 0x7ef, 0x0, 0x0, 0x7ef, 0xc00315c000, 0x7ef)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00315c000, 0x7ef, 0x7ef, 0xc00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000c525a0, 0x22, 0xc00315c000, 0x7ef, 0xc00315c000, 0x7ef, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000c525a0, 0x22, 0xc00315c000, 0x7ef, 0xaa0378, 0x1, 0xc00315c000, 0x7ef, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000478660, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 285 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc0028c76e0, 0x0, 0xc001529260)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0x0, 0xc0028c76e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:634 +0x4a
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc0028c76b0, 0xc001529320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc001529518, 0xc0034ac180, 0xc00137e600)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc001529518, 0xc0034ac180, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc001529518, 0xc0034ac180, 0xc0034ac180)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000478720, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 287 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1d4000fb0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00322a8d0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00322a8c0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0044fb500, 0xa37, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00007f080, 0x55, 0xc0044fb500, 0xa37, 0xc0044fb500, 0xa37, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00007f080, 0x55, 0xc0044fb500, 0xa37, 0xaa0378, 0x1, 0xc0044fb500, 0xa37, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004788a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 288 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc003822000, 0xfc3, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc003822000, 0xfc3, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc003822000, 0xfc3, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc003822000, 0xfc3, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc003822000, 0xfc3, 0x0, 0x0, 0xfc3, 0xc003822000, 0xfc3)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc003822000, 0xfc3, 0xfc3, 0x1000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001fee060, 0x5a, 0xc003822000, 0xfc3, 0xc003822000, 0xfc3, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001fee060, 0x5a, 0xc003822000, 0xfc3, 0xaa0378, 0x1, 0xc003822000, 0xfc3, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000478960, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 289 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc004580000, 0x1446, 0xc0017fb218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc004580000, 0x1446, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0017fb3c0, 0xaa04fd, 0x2, 0xc004580000, 0x1446, 0x1, 0x43f, 0xc00229cb00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004580000, 0x1446, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0017fb518, 0xc0010b06a0, 0x32, 0xc001287900)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000478a20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 290 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0017ed3c0, 0xaa04f9, 0x2, 0xc002664a00, 0x2354, 0x7, 0x679, 0xc00135fd40)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002664a00, 0x2354, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0017ed518, 0xc00039eb00, 0x3c, 0xc001bfba40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000478ae0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 291 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).breakLongLines(0xc0018f10c0, 0xaa04bf, 0x2, 0xba, 0x2, 0xc0018f0198)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:462 +0x58c
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0018f10c0, 0xaa04bf, 0x2, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:426 +0x63
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0018f10c0, 0xc0063ee2a0, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:298 +0xb1
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0018f10c0, 0xc0063ee2a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0018f10c0, 0xc0063ee2a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0018f10c0, 0xc0063e8e00, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0018f10c0, 0xc0063e8e00, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0018f10c0, 0xc0063e8e00, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0018f10c0, 0xc0063e8d20, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0018f10c0, 0xc0063e8d20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0018f10c0, 0xc0063e8d20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0018f10c0, 0xc0063e8620, 0xc001d7ae00, 0xb6c046)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0018f10c0, 0xc0063e8620, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0018f10c0, 0xc0063e8620, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0018f10c0, 0xc0063e8540, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0018f10c0, 0xc0063e8540, 0x54b170, 0xc0063ca4e0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc0063e8540, 0x0, 0x0, 0x0, 0x0, 0xc0063e8540, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc000e501b0, 0x0, 0x0, 0x0, 0x1800, 0xc000ec9800, 0x1765, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc000ec9800, 0x1765, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc000de6c80)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0018f1518, 0xc00135e040, 0xc00135eea0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000478ba0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 292 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc001cfc340, 0xc0052cd220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0052cd588, 0xaa04f9, 0x2, 0xc006420600, 0x129b, 0x641e2d, 0x0, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0052cd518, 0xc001dd2100, 0x52)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000478c60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 293 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe12000cb50, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0010a66a0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0010a6690, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00002c600, 0x59e, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00002a960, 0x43, 0xc00002c600, 0x59e, 0xc00002c600, 0x59e, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00002a960, 0x43, 0xc00002c600, 0x59e, 0xaa0378, 0x1, 0xc00002c600, 0x59e, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000478d20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 294 [semacquire]:
bytes.makeSlice(0xe00, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0015010e8, 0x200, 0x400)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).ReadFrom(0xc0015010e8, 0xb7b620, 0xc0028c6030, 0x203000, 0x203000, 0x203000)
/usr/local/go/src/bytes/buffer.go:202 +0x48
io/ioutil.readAll(0xb7b620, 0xc0028c6030, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:36 +0xe3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc0028c6030, 0xc0015011d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0028c6030, 0x0, 0x0, 0x0, 0x1300, 0xc006884c00, 0x11d4, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc006884c00, 0x11d4, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc0022f3680)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001501518, 0xc005bcf240, 0xc001246920)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000478de0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 295 [semacquire]:
strings.genSplit(0xc00454a380, 0xd7a, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc005402000, 0x2da, 0x2da)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000fe93c0, 0xaa0599, 0x2, 0xc001c14d80, 0xd7a, 0x1, 0x2da, 0xc00229cb80)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001c14d80, 0xd7a, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000fe9518, 0xc001cfd2c0, 0x32, 0xc0023e3440)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000478ea0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 397 [semacquire]:
strings.Replace(0xc004db3300, 0x1166, 0xaa042f, 0x2, 0xaa0af0, 0x3, 0x5, 0xc000e51200, 0x0)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:175 +0x72
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0043cf8f8, 0xc004db3300, 0x1166, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003815c0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 298 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0054ab0c0, 0xc0010f8fa0, 0x93, 0x93, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:430 +0x233
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0054ab0c0, 0xc002fc2930, 0xc0054aa1b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0054ab0c0, 0xc002fc28c0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0054ab0c0, 0xc002fc28c0, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0054ab0c0, 0xc002fc28c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0054ab0c0, 0xc002fc28c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0054ab0c0, 0xc001177d50, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0007410c0, 0xc001177d50, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007410c0, 0xc001177d50, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0054ab0c0, 0xc001177c70, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0007410c0, 0xc001177c70, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007410c0, 0xc001177c70, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0054ab0c0, 0xc001177570, 0xc000cb6a80, 0x7fe260fee9b8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0007410c0, 0xc001177570, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007410c0, 0xc001177570, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0054ab0c0, 0xc001177490, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0007410c0, 0xc001177490, 0x54b170, 0xc002eedad0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc001177490, 0x0, 0x0, 0x0, 0x0, 0xc001177490, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc00323ce40, 0x0, 0x0, 0x0, 0xa80, 0xc002b72a80, 0x967, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc002b72a80, 0x967, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc00002b9f0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000741518, 0xc00039f000, 0xc000c44e00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004790e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 299 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Join(0xc004f1a9f0, 0x3, 0x3, 0xaa0378, 0x1, 0x0, 0xc003580010)
/usr/local/go/src/strings/strings.go:438 +0x4c8
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000c01518, 0xc000fee860, 0xc0022265a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:334 +0xa85
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004791a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 300 [semacquire]:
github.com/gigawattio/window.Rolling(0xc0057f0000, 0x1a7, 0x1a7, 0x3, 0x0, 0x1a6, 0xc0057f0000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:10 +0x6c
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000e3d518, 0xc000fee360, 0xc0009c73e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000479260, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 301 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00171b3c0, 0xaa059b, 0x2, 0xc0005fe800, 0xb4a, 0x18, 0x258, 0xc001000e00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0005fe800, 0xb4a, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00171b518, 0xc000b33f60, 0x59, 0xc0004795c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000479320, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 303 [semacquire]:
regexp/syntax.(*parser).newRegexp(...)
/usr/local/go/src/regexp/syntax/parse.go:94
regexp/syntax.(*parser).newLiteral(0xc001702900, 0xd500000064, 0x1)
/usr/local/go/src/regexp/syntax/parse.go:182 +0x11f
regexp/syntax.(*parser).literal(0xc001702900, 0x64)
/usr/local/go/src/regexp/syntax/parse.go:210 +0x3b
regexp/syntax.Parse(0xabc10c, 0x11, 0xd4, 0x0, 0x0, 0xabc106)
/usr/local/go/src/regexp/syntax/parse.go:729 +0x2ae
regexp.compile(0xabc10c, 0x11, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:170 +0x5a
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0018ab300, 0x0, 0xabc106, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0018ab300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0018ab300, 0x0, 0xb856e0, 0xc00039e008, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0018ab300, 0xc001666a80, 0x0, 0x8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0018ab300, 0x42ca96, 0xc0035d02a0, 0x30, 0x28, 0xa4b3a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xabc101, 0x1d, 0xc0018ab370, 0x42d308, 0x30, 0xa4b3a0, 0xb85601)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xabc101, 0x1d, 0xc001396b40, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc000c472f0, 0xabc101, 0x1d, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc0018ab518, 0xc000fee040, 0xabc101, 0x1d, 0x36, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:133 +0x3a7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004794a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 304 [semacquire]:
strings.genSplit(0xc000d5fae0, 0x9f, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x1, 0xc000480000, 0x15)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*ContentExtractor).isHighLinkDensity(0xc001eb5518, 0xc0056e01e0, 0x2)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:567 +0xd5
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0x0, 0xc0056e01e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:636 +0xa4
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc0056e01b0, 0xc001eb5320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc001eb5518, 0xc001290a80, 0xc0009c7660)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc001eb5518, 0xc001290a80, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc001eb5518, 0xc001290a80, 0xc001290a80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000479560, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 305 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc002554700, 0x607, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:87 +0x379
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc002554700, 0x607, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc002554700, 0x607, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc002554700, 0x607, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc002554700, 0x607, 0x0, 0x0, 0x607, 0xc002554700, 0x607)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc002554700, 0x607, 0x607, 0x800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000764d20, 0x43, 0xc002554700, 0x607, 0xc002554700, 0x607, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000764d20, 0x43, 0xc002554700, 0x607, 0xaa0378, 0x1, 0xc002554700, 0x607, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000479620, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 322 [semacquire]:
github.com/gigawattio/window.Rolling(0xc00473e000, 0x166, 0x166, 0x3, 0x0, 0x165, 0xc00473e000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000b99518, 0xc000fef220, 0xc00135f520)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004796e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 323 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000e2b3c0, 0xaa0503, 0x2, 0xc00099c400, 0xbed, 0x3, 0x255, 0xc002489aa0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00099c400, 0xbed, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000e2b518, 0xc00135f620, 0x42, 0xc00002ae10)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004797a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 324 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe234003de0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0030be4b0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0030be4a0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003220000, 0xa08, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000b73a80, 0x3f, 0xc003220000, 0xa08, 0xc003220000, 0xa08, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000b73a80, 0x3f, 0xc003220000, 0xa08, 0xaa0378, 0x1, 0xc003220000, 0xa08, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000479860, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 325 [runnable]:
github.com/fatih/set.(*Set).Has(0xc00039e080, 0xc0023691c8, 0x1, 0x1, 0xc000480000)
/home/develop/go/src/github.com/fatih/set/set_ts.go:82 +0xd4
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002369400, 0xaa04f9, 0x2, 0xc00632e310, 0x64, 0xc002369240, 0x42d308, 0x8)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:81 +0x1e7
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords.func1(0x0, 0xc0068049c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:174 +0x86
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc006804990, 0xc0023692c8, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords(0xc002369388)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:173 +0xb0
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc002369388, 0xc0034581e0, 0xaa04f9, 0x2, 0xc002369518, 0x8, 0x9ecc80, 0xc002369301, 0xc000512238)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:43 +0xf5
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc002369518, 0xc0034581e0, 0xaa04f9, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000479920, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 326 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0037270c0, 0xc002a36540, 0xbc, 0xbc, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:430 +0x233
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0037270c0, 0xc00203e700, 0xc0037261b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0037270c0, 0xc00203e690, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0037270c0, 0xc00203e690, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0037270c0, 0xc00203e690, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0037270c0, 0xc00203e690, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0037270c0, 0xc000fbdce0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000b950c0, 0xc000fbdce0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000b950c0, 0xc000fbdce0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0037270c0, 0xc000fbdc00, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000b950c0, 0xc000fbdc00, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000b950c0, 0xc000fbdc00, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0037270c0, 0xc000fbd500, 0xc000b94d40, 0x46a81e)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000b950c0, 0xc000fbd500, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000b950c0, 0xc000fbd500, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0037270c0, 0xc000fbd420, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000b950c0, 0xc000fbd420, 0x54b170, 0xc002eed790)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc000fbd420, 0x0, 0x0, 0x0, 0x0, 0xc000fbd420, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc002f07a70, 0x0, 0x0, 0x0, 0xc00, 0xc002f20c00, 0xb47, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc002f20c00, 0xb47, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc0001147e0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000b95518, 0xc000b339e0, 0xc000b33e60)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004799e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 327 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0058b8000, 0x1051, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0058b8000, 0x1051, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0058b8000, 0x1051, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0058b8000, 0x1051, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0058b8000, 0x1051, 0x0, 0x0, 0x1051, 0xc0058b8000, 0x1051)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0058b8000, 0x1051, 0x1051, 0x1500)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000ccc040, 0x40, 0xc0058b8000, 0x1051, 0xc0058b8000, 0x1051, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000ccc040, 0x40, 0xc0058b8000, 0x1051, 0xaa0378, 0x1, 0xc0058b8000, 0x1051, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000479aa0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 328 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0044c5800, 0x6bd, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x2e6
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0044c5800, 0x6bd, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0044c5800, 0x6bd, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0044c5800, 0x6bd, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0044c5800, 0x6bd, 0x0, 0x0, 0x6bd, 0xc0044c5800, 0x6bd)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0044c5800, 0x6bd, 0x6bd, 0x700)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00002a870, 0x4a, 0xc0044c5800, 0x6bd, 0xc0044c5800, 0x6bd, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00002a870, 0x4a, 0xc0044c5800, 0x6bd, 0xaa0378, 0x1, 0xc0044c5800, 0x6bd, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000479b60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 329 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003405588, 0xaa04f9, 0x2, 0xc00599ad90, 0x66, 0x5, 0x13, 0xc00039fd60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc003405518, 0xc001246320, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000479c20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 330 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc004cdb980, 0xc000f7d2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000f7d3c0, 0xaa058b, 0x2, 0xc00360d300, 0x128c, 0x7, 0x3f2, 0xc00103ea60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00360d300, 0x128c, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000f7d518, 0xc0010b1c00, 0x44, 0xc0011261f1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000479ce0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 331 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000b79588, 0xaa04f9, 0x2, 0xc0040f9ba0, 0xc4, 0x0, 0xc000b793d8, 0xc0000b2408)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc000b79518, 0xc001b62510, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:528 +0xfd
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc000b79518, 0xc000c44320, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000479da0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 333 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe13000bc20, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000a7ca40, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc000a7ca30, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000433400, 0x4b0, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000b72780, 0x3d, 0xc000433400, 0x4b0, 0xc000433400, 0x4b0, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000b72780, 0x3d, 0xc000433400, 0x4b0, 0xaa0378, 0x1, 0xc000433400, 0x4b0, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000479f20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1059 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0042e13c0, 0xaa0503, 0x2, 0xc001e21000, 0x2b78, 0x8, 0x864, 0xc0000b3c60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001e21000, 0x2b78, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0042e1518, 0xc003113e00, 0x41, 0xc004af8b90)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fefda0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 309 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1d4227600, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00316c570, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00316c550, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000660c00, 0xb60, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000cd4960, 0x2c, 0xc000660c00, 0xb60, 0xc000660c00, 0xb60, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000cd4960, 0x2c, 0xc000660c00, 0xb60, 0xaa0378, 0x1, 0xc000660c00, 0xb60, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f740, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 418 [semacquire]:
regexp.(*Regexp).get(0xc000149ae0, 0x12a05f200)
/usr/local/go/src/regexp/regexp.go:255 +0x247
regexp.(*Regexp).doExecute(0xc000149ae0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0022d9802, 0x15c5, 0x0, 0x2, ...)
/usr/local/go/src/regexp/exec.go:538 +0xb5
regexp.(*Regexp).replaceAll(0xc000149ae0, 0x0, 0x0, 0x0, 0xc0022d9802, 0x15c5, 0x2, 0xc0031cda58, 0xaa3a2a, 0x4, ...)
/usr/local/go/src/regexp/regexp.go:599 +0x1a5
regexp.(*Regexp).ReplaceAllString(0xc000149ae0, 0xc0022d9802, 0x15c5, 0x0, 0x0, 0xc00038e060, 0x127d7a0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
_/home/ec2-user/environment/cluster2/cluster2/base.Sanitize(0xaa3a2a, 0x4, 0xc0022d9802, 0x15c5, 0x0, 0x0, 0x10, 0xab52f4)
/home/ec2-user/environment/cluster2/cluster2/base/sanitizer.go:44 +0xc9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000380ba0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:304 +0x33a
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 310 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1b022a390, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002d7a510, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002d7a500, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0048b2300, 0x2d7, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0016a9e00, 0x41, 0xc0048b2300, 0x2d7, 0xc0048b2300, 0x2d7, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0016a9e00, 0x41, 0xc0048b2300, 0x2d7, 0xaa0378, 0x1, 0xc0048b2300, 0x2d7, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043f8c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 321 [semacquire]:
regexp.makeOnePass(0xc0035d09c0, 0xc0035d09c0)
/usr/local/go/src/regexp/onepass.go:309 +0x20e
regexp.compileOnePass(0xc0035d0930, 0xc0035d0930)
/usr/local/go/src/regexp/onepass.go:500 +0x129
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc004478ff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc004478ff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc0062884f4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc004478ff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc004478ff8, 0x1, 0xc00407a010, 0xc0104303101, 0x280460fef080)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc004478ff8, 0x4, 0x192, 0xc001c81e01, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc001c81000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc000236a90, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0035d0900, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0044798f8, 0xc003ae1000, 0xe61, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fefaa0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1058 [semacquire]:
bytes.makeSlice(0xf6a, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc004383388, 0x14a, 0x139)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc004383388, 0xc0006f5e40, 0x14a, 0x5165d7, 0xc0062166fd, 0xc000c91120)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00575e620)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00575e5b0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00574b8f0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00574b810)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00574b110)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc004c3b830, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004383518, 0xc002408820, 0x63, 0xc001d2ae70)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fefc20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 382 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00115b820, 0xc0043d92c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0043d93c0, 0xaa0503, 0x2, 0xc004b84400, 0x804, 0x3, 0x176, 0xc004c39820)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004b84400, 0x804, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0043d9518, 0xc004c39440, 0x54, 0xc002b2b020)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717f20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 406 [semacquire]:
github.com/PuerkitoBio/goquery.appendWithoutDuplicates(0xc005a653c0, 0x4, 0x4, 0xc003b379e0, 0x6, 0x6, 0xc00387d260, 0xc005a65380, 0x20, 0x20)
/home/develop/go/src/github.com/PuerkitoBio/goquery/utilities.go:137 +0x1cf
github.com/PuerkitoBio/goquery.mapNodes(0xc000889c80, 0x1, 0x1, 0xc00387d330, 0x1, 0x0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:694 +0x159
github.com/PuerkitoBio/goquery.findWithMatcher(0xc000889c80, 0x1, 0x1, 0xb856e0, 0xc005a65380, 0xc00387d3b0, 0x753f20, 0xaa3fc2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:543 +0x81
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc006537020, 0xaa408e, 0x4, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x85
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00387d518, 0xc001885b20, 0x3e, 0xc0021770c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:126 +0x200
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061f800, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 320 [semacquire]:
golang.org/x/net/html.(*parser).addFormattingElement(0xc000236410)
/home/develop/go/src/golang.org/x/net/html/parse.go:378 +0x552
golang.org/x/net/html.inBodyIM(0xc000236410, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:981 +0x7b1
golang.org/x/net/html.(*parser).parseCurrentToken(0xc000236410)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc000236410, 0xc0032318f0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc000b32000, 0x0, 0x0, 0x0, 0xc002f4f200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc000b32000, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc002f4f8f8, 0xc001204000, 0x972, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef920, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 319 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc003bd0340, 0xc006289568, 0x8)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc003bd0340, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc003bd0340)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc003bd0340, 0xc003c1e070, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0034d1a00, 0x0, 0x0, 0x0, 0xc004281200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0034d1a00, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0042818f8, 0xc003850a00, 0x19ef, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef7a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 316 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc004f59900, 0x21c6, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:153 +0x1f1
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc004f59900, 0x21c6, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc004f59900, 0x21c6, 0x0, 0x0, 0x21c6, 0xc004f59900, 0x21c6)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc004f59900, 0x21c6, 0x21c6, 0x2500)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc004643180, 0x4c, 0xc004f59900, 0x21c6, 0xc004f59900, 0x21c6, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc004643180, 0x4c, 0xc004f59900, 0x21c6, 0xaa0378, 0x1, 0xc004f59900, 0x21c6, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef260, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 317 [semacquire]:
strings.genSplit(0xc0054de700, 0x6c7, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x42fa20, 0x183, 0x183)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003c8d3c0, 0xaa0569, 0x2, 0xc005bf5500, 0x6c7, 0x36, 0x183, 0xc005bce880)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005bf5500, 0x6c7, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003c8d518, 0xc004cda0a0, 0x46, 0xc00456c057)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef3e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 314 [semacquire]:
strings.genSplit(0xc00282b500, 0x983, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc002574a00, 0x145, 0x145)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002ac13c0, 0xaa058b, 0x2, 0xc003f6a000, 0x983, 0x7, 0x145, 0xc001396d20)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003f6a000, 0x983, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002ac1518, 0xc00039e640, 0x4f, 0xc00277c0a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001feed20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 318 [semacquire]:
bytes.makeSlice(0xe52, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc001b62480, 0x1b4, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc001b62480, 0xc002e92700, 0x1b4, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc001b62480, 0xc002e92700, 0x1b4, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc001b62480, 0xc004669c70, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc001b62480, 0xc004669c00, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001b62480, 0xc004669340, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001b62480, 0xc004669260, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001b62480, 0xc004668b60, 0xb84d20, 0xc001b62480)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc001b62480, 0xc004668b60, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0004d0630, 0x193b9e627, 0x1251a20, 0xc004cda1a0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef620, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 315 [semacquire]:
github.com/gigawattio/window.Rolling(0xc000b34000, 0x370, 0x370, 0x3, 0x0, 0x36f, 0xc000b34000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001bef518, 0xc00039ef80, 0xc005ca62a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001feeea0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1107 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc000a22000, 0x613, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x2e6
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc000a22000, 0x613, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc000a22000, 0x613, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc000a22000, 0x613, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc000a22000, 0x613, 0x0, 0x0, 0x613, 0xc000a22000, 0x613)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc000a22000, 0x613, 0x613, 0x800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000115cb0, 0x20, 0xc000a22000, 0x613, 0xc000a22000, 0x613, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000115cb0, 0x20, 0xc000a22000, 0x613, 0xaa0378, 0x1, 0xc000a22000, 0x613, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061fd40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 407 [semacquire]:
bytes.makeSlice(0x40f, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0052490c0, 0x1, 0x1)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0052490c0, 0x1189d4f, 0x1, 0x1, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0052490c0, 0xc00497f400, 0x1e4, 0x7, 0xc00497f400)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:440 +0xf7
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0052490c0, 0xc000457570, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:176 +0xb28
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0052490c0, 0xc000457570, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052490c0, 0xc000457340, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0021d70c0, 0xc000457340, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021d70c0, 0xc000457340, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052490c0, 0xc000457260, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0021d70c0, 0xc000457260, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021d70c0, 0xc000457260, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052490c0, 0xc003e4c230, 0xc0039fea80, 0x7fe260fef080)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0021d70c0, 0xc003e4c230, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021d70c0, 0xc003e4c230, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052490c0, 0xc003e4c150, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021d70c0, 0xc003e4c150, 0x54b170, 0xc002eec680)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc003e4c150, 0x0, 0x0, 0x0, 0x0, 0xc003e4c150, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0047dabd0, 0x0, 0x0, 0x0, 0xc00, 0xc001430c00, 0xafd, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc001430c00, 0xafd, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc0012bd260)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0021d7518, 0xc003383e00, 0xc000762ea0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061f980, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 311 [semacquire]:
strings.genSplit(0xc0039db800, 0x5f1, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x1, 0xc000480001, 0x132)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*ContentExtractor).isHighLinkDensity(0xc002fb5518, 0xc005b3f9e0, 0x2)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:567 +0xd5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc002fb5518, 0xc00137ecc0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:386 +0x1ae
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043fa40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 312 [semacquire]:
regexp/syntax.(*Regexp).CapNames(0xc006065260, 0x0, 0xd4, 0xc006065260)
/usr/local/go/src/regexp/syntax/regexp.go:308 +0x56
regexp.compile(0xabc10c, 0x11, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:175 +0x98
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc001365300, 0x0, 0xabc106, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc001365300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc001365300, 0x0, 0xb856e0, 0xc00039e048, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc001365300, 0xc0007f3260, 0x0, 0x8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc001365300, 0x42ca96, 0xc00649a240, 0x30, 0x28, 0xa4b3a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xabc101, 0x1d, 0xc001365370, 0x42d308, 0x30, 0xa4b3a0, 0xb85601)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xabc101, 0x1d, 0xc001cfdae0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0012bad50, 0xabc101, 0x1d, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc001365518, 0xc001397bc0, 0xabc101, 0x1d, 0x4a, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:133 +0x3a7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043fbc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 313 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc00151d450, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x260
golang.org/x/net/html.(*parser).parse(0xc00151d520, 0xc0052da070, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7b620, 0xc000cca0c0, 0x0, 0x0, 0x0, 0x0, 0x701, 0xc0034ad440)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0034ad440, 0x0, 0x0, 0x0, 0x800, 0xc004756800, 0x74a, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:93 +0x76
github.com/jaytaylor/html2text.FromString(0xc004756800, 0x74a, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc0008a0cf0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0039d1518, 0xc00137f800, 0xc00115a200)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043fd40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 377 [semacquire]:
bytes.makeSlice(0x1d44, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc000e29388, 0x1308, 0x213)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc000e29388, 0xc00231d500, 0x1308, 0x5165d7, 0xc00546644f, 0xc0033a5b50)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0009b96c0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc004bd92d0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc004bd91f0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc004bd8540)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc004bd8460)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc004b2e930)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc004d5d6b0, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000e29518, 0xc001cfdce0, 0x36, 0xc003264200)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043fec0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 378 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe15c008ed0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0038b76a0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0038b7690, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00015c800, 0x3bd, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000e85c00, 0x3a, 0xc00015c800, 0x3bd, 0xc00015c800, 0x3bd, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000e85c00, 0x3a, 0xc00015c800, 0x3bd, 0xaa0378, 0x1, 0xc00015c800, 0x3bd, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000380900, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 380 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Join(0xc003f72b10, 0x3, 0x3, 0xaa0378, 0x1, 0x0, 0xc002f08860)
/usr/local/go/src/strings/strings.go:438 +0x4c8
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0017f5518, 0xc005048900, 0xc005049140)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:330 +0x9c5
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000380ae0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 360 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe174001090, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0012a0970, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0012a0960, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002aff400, 0x3e1, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000114150, 0x2e, 0xc002aff400, 0x3e1, 0xc002aff400, 0x3e1, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000114150, 0x2e, 0xc002aff400, 0x3e1, 0xaa0378, 0x1, 0xc002aff400, 0x3e1, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061e000, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 361 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe16c005880, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0008c9b80, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0008c9b70, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000afe900, 0x429, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000bbac80, 0x47, 0xc000afe900, 0x429, 0xc000afe900, 0x429, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000bbac80, 0x47, 0xc000afe900, 0x429, 0xaa0378, 0x1, 0xc000afe900, 0x429, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061e0c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 363 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1c41d5380, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000351910, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0003518f0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000a0b300, 0x11f4, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000ccdb40, 0x3f, 0xc000a0b300, 0x11f4, 0xc000a0b300, 0x11f4, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000ccdb40, 0x3f, 0xc000a0b300, 0x11f4, 0xaa0378, 0x1, 0xc000a0b300, 0x11f4, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061e240, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 364 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe190008fa0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0013e3410, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0013e3400, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000afed80, 0x44a, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000765db0, 0x44, 0xc000afed80, 0x44a, 0xc000afed80, 0x44a, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000765db0, 0x44, 0xc000afed80, 0x44a, 0xaa0378, 0x1, 0xc000afed80, 0x44a, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061e300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 365 [semacquire]:
fmt.Errorf(0xabb2ed, 0x1c, 0xc0034510c8, 0x1, 0x1, 0xa0, 0xc005e1c890)
/usr/local/go/src/fmt/errors.go:21 +0xa3
github.com/araddon/dateparse.unknownErr(...)
/home/develop/go/src/github.com/araddon/dateparse/parseany.go:117
github.com/araddon/dateparse.parseTime(0xc00632dea0, 0x93, 0x0, 0xc00632dea0, 0x93, 0x93)
/home/develop/go/src/github.com/araddon/dateparse/parseany.go:237 +0x2cc
github.com/araddon/dateparse.ParseAny(0xc00632dea0, 0x93, 0x3, 0xaa0378, 0x1, 0xc00632dea0, 0x93)
/home/develop/go/src/github.com/araddon/dateparse/parseany.go:124 +0x42
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00350f518, 0xc00115a220, 0xc00115abc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:344 +0xbce
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061e3c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 366 [semacquire]:
github.com/fatih/set.newTS(...)
/home/develop/go/src/github.com/fatih/set/set_ts.go:16
github.com/fatih/set.New(...)
/home/develop/go/src/github.com/fatih/set/set.go:56
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000c353c0, 0xaa04d9, 0x2, 0xc0040f5000, 0x7e1, 0x4, 0x1b4, 0xc0022263e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:74 +0x61
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0040f5000, 0x7e1, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000c35518, 0xc00115ac80, 0x3d, 0xc003492286)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061e480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 367 [semacquire]:
github.com/gigawattio/window.Rolling(0xc00179d900, 0x130, 0x130, 0x3, 0x0, 0x12f, 0xc00179d900)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003ea5518, 0xc00115afa0, 0xc002226c20)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061e540, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 368 [semacquire]:
strings.genSplit(0xc00399bc00, 0x689, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x0, 0xc00240a380, 0x10000c002e3f360)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002e3f3c0, 0xaa04f9, 0x2, 0xc00399b500, 0x689, 0xc00399b500, 0xc002e3f3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00399b500, 0x689, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002e3f518, 0xc00115b500, 0x4e, 0xc0007659f0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061e600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 369 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc00215c880, 0xd03, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc00215c880, 0xd03, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc00215c880, 0xd03, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc00215c880, 0xd03, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc00215c880)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00215c880, 0xd03, 0x0, 0x0, 0xd03, 0xd80, 0xc00215c880)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00215c880, 0xd03, 0xd03, 0xd80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc003492d40, 0x34, 0xc00215c880, 0xd03, 0xc00215c880, 0xd03, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc003492d40, 0x34, 0xc00215c880, 0xd03, 0xaa0378, 0x1, 0xc00215c880, 0xd03, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061e6c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 435 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1d01aa170, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0001b6d80, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0001b6d70, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003b25800, 0x591, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000c50680, 0x3d, 0xc003b25800, 0x591, 0xc003b25800, 0x591, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000c50680, 0x3d, 0xc003b25800, 0x591, 0xaa0378, 0x1, 0xc003b25800, 0x591, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061e840, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 436 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc0033a89c0, 0xc0010f86e0, 0x9d)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc0033a89c0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0033a89c0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0033a89c0, 0xc00549d030, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7b620, 0xc0010a49c0, 0x0, 0x0, 0x0, 0x0, 0xa01, 0xc0010a4990)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0010a4990, 0x0, 0x0, 0x0, 0xa80, 0xc002e82a80, 0xa25, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:93 +0x76
github.com/jaytaylor/html2text.FromString(0xc002e82a80, 0xa25, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0xc003e3cfb8, 0x75bd50)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002393518, 0xc00137f460, 0xc005049c40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061e900, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 437 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1280059a0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0001de590, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0001de570, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00380a800, 0x3bd, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00235ddc0, 0x37, 0xc00380a800, 0x3bd, 0xc00380a800, 0x3bd, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00235ddc0, 0x37, 0xc00380a800, 0x3bd, 0xaa0378, 0x1, 0xc00380a800, 0x3bd, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061e9c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 438 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1c41ec0d0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0028a6110, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0028a6100, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002f2d400, 0x490, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00235da40, 0x3e, 0xc002f2d400, 0x490, 0xc002f2d400, 0x490, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00235da40, 0x3e, 0xc002f2d400, 0x490, 0xaa0378, 0x1, 0xc002f2d400, 0x490, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061ea80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 439 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc002252000, 0x399, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:87 +0x379
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc002252000, 0x399, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc002252000, 0x399, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc002252000, 0x399, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc002252000, 0x399, 0x0, 0x0, 0x399, 0xc002252000, 0x399)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc002252000, 0x399, 0x399, 0x400)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0008a1740, 0x2f, 0xc002252000, 0x399, 0xc002252000, 0x399, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0008a1740, 0x2f, 0xc002252000, 0x399, 0xaa0378, 0x1, 0xc002252000, 0x399, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061eb40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 440 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc001c0e000, 0x1e0c, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc001c0e000, 0x1e0c, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc001c0e000, 0x1e0c, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc001c0e000, 0x1e0c, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc001c0e000, 0x1e0c, 0x0, 0x0, 0x1e0c, 0xc001c0e000, 0x1e0c)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc001c0e000, 0x1e0c, 0x1e0c, 0x2500)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00189b3ba, 0x42, 0xc001c0e000, 0x1e0c, 0xc001c0e000, 0x1e0c, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00189b3ba, 0x42, 0xc001c0e000, 0x1e0c, 0xaa0378, 0x1, 0xc001c0e000, 0x1e0c, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061ec00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 441 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1c4000c60, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0035ed0b0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0035ed0a0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003b6b800, 0x608, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000dacdb0, 0x2b, 0xc003b6b800, 0x608, 0xc003b6b800, 0x608, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000dacdb0, 0x2b, 0xc003b6b800, 0x608, 0xaa0378, 0x1, 0xc003b6b800, 0x608, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061ecc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 442 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.LangToStringShort(0xf, 0x5da, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesUtil.go:199 +0x56
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Lang.Iso6391(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesUtil.go:191
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc002aacc00, 0x5da, 0x5da, 0x700)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x4b
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00226aa00, 0x42, 0xc002aacc00, 0x5da, 0xc002aacc00, 0x5da, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00226aa00, 0x42, 0xc002aacc00, 0x5da, 0xaa0378, 0x1, 0xc002aacc00, 0x5da, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061ed80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 443 [semacquire]:
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc001050b40, 0xbf, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:560 +0x135
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0052c50c0, 0xc000214460, 0xc0052c41b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052c50c0, 0xc0002143f0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0052c50c0, 0xc0002143f0, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0052c50c0, 0xc0002143f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0052c50c0, 0xc0002143f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052c50c0, 0xc0020121c0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0022b70c0, 0xc0020121c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0022b70c0, 0xc0020121c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052c50c0, 0xc0020120e0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0022b70c0, 0xc0020120e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0022b70c0, 0xc0020120e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052c50c0, 0xc004979110, 0x7fe23814c7c0, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0022b70c0, 0xc004979110, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0022b70c0, 0xc004979110, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052c50c0, 0xc004979030, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0022b70c0, 0xc004979030, 0x54b170, 0xc002ffa820)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc004979030, 0x0, 0x0, 0x0, 0x0, 0xc004979030, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc00323ab40, 0x0, 0x0, 0x0, 0xc80, 0xc0017c3900, 0xc0a, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0017c3900, 0xc0a, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc0008a1860)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0022b7518, 0xc002f71a00, 0xc00522d560)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061ee40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 444 [semacquire]:
github.com/gigawattio/window.Rolling(0xc004b98000, 0x2d0, 0x2d0, 0x3, 0x0, 0x2cf, 0xc004b98000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00173b518, 0xc002227f40, 0xc0007628c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061ef00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 445 [semacquire]:
strings.Replace(0xc00477b000, 0x2b85, 0xaa03a3, 0x1, 0xaa0378, 0x1, 0x4a, 0xc00477b000, 0x2b85)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0031cd518, 0xc00229d700, 0xc000c441a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:318 +0x708
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061efc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 412 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc004b3c000, 0x7be, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc004b3c000, 0x7be, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc004b3c000, 0x7be, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc004b3c000, 0x7be, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc004b3c000, 0x7be, 0x0, 0x0, 0x7be, 0xc004b3c000, 0x7be)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc004b3c000, 0x7be, 0x7be, 0x900)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001fb6540, 0x4a, 0xc004b3c000, 0x7be, 0xc004b3c000, 0x7be, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001fb6540, 0x4a, 0xc004b3c000, 0x7be, 0xaa0378, 0x1, 0xc004b3c000, 0x7be, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000381d40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 447 [runnable]:
sync.(*RWMutex).RUnlock(0xc00039e208)
/usr/local/go/src/sync/rwmutex.go:62 +0x50
github.com/fatih/set.(*Set).Has(0xc00039e200, 0xc002a6f2c0, 0x1, 0x1, 0xc000480000)
/home/develop/go/src/github.com/fatih/set/set_ts.go:86 +0x10c
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002a6f3c0, 0xaa04d9, 0x2, 0xc002f59300, 0x1939, 0x1, 0x4b4, 0xc00135fc00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:81 +0x1e7
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002f59300, 0x1939, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002a6f518, 0xc0000b2b00, 0x52, 0xc00061e4e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061f140, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 448 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe13c000e90, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002de07b0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002de07a0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002e52e00, 0x33f, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000114360, 0x2a, 0xc002e52e00, 0x33f, 0xc002e52e00, 0x33f, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000114360, 0x2a, 0xc002e52e00, 0x33f, 0xaa0378, 0x1, 0xc002e52e00, 0x33f, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061f200, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 449 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe234003440, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00118dc20, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00118dc10, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003312480, 0x471, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000ccde00, 0x3e, 0xc003312480, 0x471, 0xc003312480, 0x471, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000ccde00, 0x3e, 0xc003312480, 0x471, 0xaa0378, 0x1, 0xc003312480, 0x471, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061f2c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 450 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc0013fc000, 0x3bab, 0xc003557218, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc0013fc000, 0x3bab, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0035573c0, 0xaa04fd, 0x2, 0xc0013fc000, 0x3bab, 0x1, 0xaab, 0xc000763c00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0013fc000, 0x3bab, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003557518, 0xc001001fa0, 0x59, 0xc001835620)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061f4a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 451 [semacquire]:
strings.genSplit(0xc0054b8000, 0x489d, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc005de2000, 0x122f, 0x122f)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0039cb3c0, 0xaa0569, 0x2, 0xc0015d4a80, 0x489d, 0xd8, 0x122f, 0xc002300ee0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0015d4a80, 0x489d, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0039cb518, 0xc002408380, 0x3d, 0xc00087a600)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061f560, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 452 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc004377588, 0xaa04f9, 0x2, 0xc004c77500, 0x131e, 0x0, 0xc002ec7880, 0x7fe260fef080)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc004377518, 0xc003382840, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061f620, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 453 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc0010b14a0, 0xc0031512c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0031513c0, 0xaa04f9, 0x2, 0xc00339ca80, 0x9e6, 0x1, 0x1f3, 0xc0007631e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00339ca80, 0x9e6, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003151518, 0xc0034d1940, 0x46, 0xc000ea6cd0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061f6e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 454 [semacquire]:
regexp.(*Regexp).ReplaceAllString(0xc000149540, 0xc0041a7800, 0x7b6, 0xaa04bf, 0x2, 0x1, 0x1)
/usr/local/go/src/regexp/regexp.go:560 +0x135
github.com/jaytaylor/html2text.FromHTMLNode(0xc00294ff80, 0x0, 0x0, 0x0, 0x0, 0xc00294ff80, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:80 +0x191
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc005222240, 0x0, 0x0, 0x0, 0xc00, 0xc001deb400, 0xb1a, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc001deb400, 0xb1a, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000a7e600)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001715518, 0xc003308e80, 0xc000b33c60)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061f7a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 455 [semacquire]:
github.com/gigawattio/window.Rolling(0xc00470e000, 0x2b7, 0x2b7, 0x3, 0x0, 0x2b6, 0xc00470e000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003559518, 0xc0034d1f80, 0xc003383d40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061f860, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 456 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc002a38cf0, 0x203000, 0x203000, 0x49d6a39b00000000, 0xc00033b138)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs.func1(0x0, 0xc002a38cf0)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:496 +0x67
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc002a38cc0, 0xc00033b2c8, 0x7)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs(0xc00033b6b0, 0xc0018857c0, 0xaac03e, 0x7, 0xc0018857c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:495 +0xd8
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc00033b6b0, 0xc0018857c0, 0xc00000ea00)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:302 +0x39d
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061f920, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 457 [semacquire]:
strings.genSplit(0xc001b86000, 0x3b56, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc006122000, 0xfdc, 0xfdc)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00314d3c0, 0xaa04df, 0x2, 0xc002ec8000, 0x3b56, 0x1, 0xfdc, 0xc005d7a4a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002ec8000, 0x3b56, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00314d518, 0xc001885d40, 0x27, 0xc000b6d2f0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061f9e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 458 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0023ac800, 0x7f7, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:59 +0x21d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0023ac800, 0x7f7, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0023ac800, 0x7f7, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0023ac800, 0x7f7, 0x0, 0x0, 0x7f7, 0xc0023ac800, 0x7f7)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0023ac800, 0x7f7, 0x7f7, 0x800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002dde480, 0x28, 0xc0023ac800, 0x7f7, 0xc0023ac800, 0x7f7, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002dde480, 0x28, 0xc0023ac800, 0x7f7, 0xaa0378, 0x1, 0xc0023ac800, 0x7f7, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061faa0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 459 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1e021f190, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001219ab0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001219aa0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000482700, 0x613, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000a9f780, 0x32, 0xc000482700, 0x613, 0xc000482700, 0x613, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000a9f780, 0x32, 0xc000482700, 0x613, 0xaa0378, 0x1, 0xc000482700, 0x613, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061fb60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 460 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc003113320, 0xc000333220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000333588, 0xaa04f9, 0x2, 0xc0045db000, 0x388, 0xb, 0x21, 0xc0004e77c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc000333518, 0xc0033092c0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061fc20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 461 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc001d8a900, 0x827, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x2e6
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc001d8a900, 0x827, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc001d8a900, 0x827, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc001d8a900, 0x827, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc001d8a900, 0x827, 0x0, 0x0, 0x827, 0xc001d8a900, 0x827)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc001d8a900, 0x827, 0x827, 0x900)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc003356700, 0x77, 0xc001d8a900, 0x827, 0xc001d8a900, 0x827, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc003356700, 0x77, 0xc001d8a900, 0x827, 0xaa0378, 0x1, 0xc001d8a900, 0x827, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061fce0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 462 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc000531600, 0x900, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc000531600, 0x900, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc000531600, 0x900, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc000531600, 0x900, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc000531600, 0x900, 0x0, 0x0, 0x900, 0xc000531600, 0x900)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc000531600, 0x900, 0x900, 0xa80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0016a8000, 0x45, 0xc000531600, 0x900, 0xc000531600, 0x900, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0016a8000, 0x45, 0xc000531600, 0x900, 0xaa0378, 0x1, 0xc000531600, 0x900, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061fda0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 463 [semacquire]:
strings.genSplit(0xc001261300, 0x1018, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0057c9500, 0x32c, 0x32c)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000db53c0, 0xaa058b, 0x2, 0xc002b04c00, 0x1018, 0x7, 0x32c, 0xc001000260)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002b04c00, 0x1018, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000db5518, 0xc0000b3280, 0x54, 0xc00087d080)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061fe60, 0x58)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 464 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc0028736c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc002873790, 0xc00065f0a0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0000b3e60, 0x0, 0x0, 0x0, 0xc0040df200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0000b3e60, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0040df8f8, 0xc001df3000, 0x1592, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061ff20, 0x58)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 465 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc0029ff000, 0xe5e, 0xc00246b218, 0xc001d7aa80)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc0029ff000, 0xe5e, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00246b3c0, 0xaa058b, 0x2, 0xc0029ff000, 0xe5e, 0x6, 0x2cd, 0xc0010009a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0029ff000, 0xe5e, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00246b518, 0xc003308e00, 0x32, 0xc000a9e840)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068c000, 0x58)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 466 [runnable]:
github.com/fatih/set.(*Set).Has(0xc00039e100, 0xc003ea72c0, 0x1, 0x1, 0xc000480000)
/home/develop/go/src/github.com/fatih/set/set_ts.go:82 +0xd4
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003ea73c0, 0xaa0503, 0x2, 0xc00106c000, 0xca9, 0x5, 0x27a, 0xc005964f20)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:81 +0x1e7
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00106c000, 0xca9, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003ea7518, 0xc003308500, 0x45, 0xc000ea79a7)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068c0c0, 0x58)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 467 [semacquire]:
strings.genSplit(0xc004aab800, 0x5d0, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc002e78800, 0x162, 0x162)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001a8f3c0, 0xaa04fd, 0x2, 0xc00441d800, 0x5d0, 0x1, 0x162, 0xc005964ee0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00441d800, 0x5d0, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001a8f518, 0xc003308080, 0x32, 0xc000aa9a40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068c180, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 468 [semacquire]:
strings.genSplit(0xc003ddf980, 0x195b, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc005d4ea80, 0x4a6, 0x4a6)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0045133c0, 0xaa059b, 0x2, 0xc0040ce000, 0x195b, 0x1e, 0x4a6, 0xc005b74b20)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0040ce000, 0x195b, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004513518, 0xc003309ca0, 0x36, 0xc000a9f980)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068c240, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 469 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc0026fbe10, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:901 +0x820
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0026fbe10)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0026fbe10, 0xc004334700, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7b620, 0xc004d86540, 0x0, 0x0, 0x0, 0x0, 0x1501, 0xc004d86510)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc004d86510, 0x0, 0x0, 0x0, 0x1800, 0xc004bc0000, 0x1556, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:93 +0x76
github.com/jaytaylor/html2text.FromString(0xc004bc0000, 0x1556, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc00112d620)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00171d518, 0xc0033091e0, 0xc003309b80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068c300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 470 [semacquire]:
strings.Replace(0xc004d6a700, 0x1abb, 0xaa03a3, 0x1, 0xaa0378, 0x1, 0x2c, 0xc004d6a700, 0x1abb)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003f23518, 0xc003309fe0, 0xc003212bc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:318 +0x708
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068c3c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 471 [semacquire]:
bytes.makeSlice(0x2e3, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00568a750, 0x1, 0x2)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteByte(0xc00568a750, 0xc00568a722, 0xc0010bf778, 0x8)
/usr/local/go/src/bytes/buffer.go:267 +0x8c
golang.org/x/net/html.render1(0xb84d20, 0xc00568a750, 0xc005528620, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:172 +0x3d5
golang.org/x/net/html.render1(0xb84d20, 0xc00568a750, 0xc0055281c0, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00568a750, 0xc005528150, 0xb84d20, 0xc00568a750)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc00568a750, 0xc005528150, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc002d61a40, 0x17ed94937, 0x1251a20, 0xc00522ce20, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068c480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 472 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe17c001640, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00241e030, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00241e020, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001708c00, 0xb12, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0023e2040, 0x37, 0xc001708c00, 0xb12, 0xc001708c00, 0xb12, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0023e2040, 0x37, 0xc001708c00, 0xb12, 0xaa0378, 0x1, 0xc001708c00, 0xb12, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068c540, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 473 [semacquire]:
bytes.makeSlice(0xedb, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00568aa80, 0x153, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00568aa80, 0xc004145340, 0x153, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc00568aa80, 0xc004145340, 0x153, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc00568aa80, 0xc00552eee0, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc00568aa80, 0xc00552ee70, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00568aa80, 0xc00552e5b0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00568aa80, 0xc00552e4d0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00568aa80, 0xc0012b3180, 0xb84d20, 0xc00568aa80)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc00568aa80, 0xc0012b3180, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc00568a8d0, 0x17edbdbe7, 0x1251a20, 0xc0009c65c0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068c600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 474 [semacquire]:
strings.Replace(0xc001652000, 0x1be2, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0x14, 0xc001652000, 0x1be2)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0031bd8f8, 0xc001940000, 0x1be0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068c6c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 475 [semacquire]:
github.com/jaytaylor/html2text.FromString(0xc002d5d500, 0x949, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000aa9940)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:102 +0x42
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003e83518, 0xc002409ee0, 0xc00522d240)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068c780, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 477 [semacquire]:
golang.org/x/net/html.(*Tokenizer).readTag(0xc003d5d790, 0x54e501)
/home/develop/go/src/golang.org/x/net/html/token.go:834 +0x162
golang.org/x/net/html.(*Tokenizer).readStartTag(0xc003d5d790, 0x61)
/home/develop/go/src/golang.org/x/net/html/token.go:779 +0x45
golang.org/x/net/html.(*Tokenizer).Next(0xc003d5d790, 0x1)
/home/develop/go/src/golang.org/x/net/html/token.go:1021 +0x28c
golang.org/x/net/html.(*parser).parse(0xc003d5d860, 0xc003993ce0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2284 +0xad
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0009c6060, 0x0, 0x0, 0x0, 0xc0026d3200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0009c6060, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0026d38f8, 0xc003f6b800, 0x1519, 0x20300000000000, 0x7fe23a6bafff, 0x400)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068c900, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 478 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc004028c00, 0x12f6, 0xc001813218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc004028c00, 0x12f6, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0018133c0, 0xaa058b, 0x2, 0xc004028c00, 0x12f6, 0x5, 0x3a9, 0xc00103fe00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004028c00, 0x12f6, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001813518, 0xc0027ef700, 0x45, 0xc004af8050)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068c9c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 479 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectScript(0xc00002cc00, 0x5c3, 0x2)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:271 +0x3a0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00002cc00, 0x5c3, 0x0, 0x0, 0x5c3, 0xc00002cc00, 0x5c3)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:58 +0x39
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00002cc00, 0x5c3, 0x5c3, 0x600)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000c52d80, 0x2b, 0xc00002cc00, 0x5c3, 0xc00002cc00, 0x5c3, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000c52d80, 0x2b, 0xc00002cc00, 0x5c3, 0xaa0378, 0x1, 0xc00002cc00, 0x5c3, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068ca80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 480 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1f020c9a0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0029cab90, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0029cab80, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00299ea80, 0x941, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0001156b0, 0x2a, 0xc00299ea80, 0x941, 0xc00299ea80, 0x941, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0001156b0, 0x2a, 0xc00299ea80, 0x941, 0xaa0378, 0x1, 0xc00299ea80, 0x941, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068cb40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 483 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2080017b0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0032fe030, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0032fe020, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000660000, 0xa8f, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000dac000, 0x2e, 0xc000660000, 0xa8f, 0xc000660000, 0xa8f, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000dac000, 0x2e, 0xc000660000, 0xa8f, 0xaa0378, 0x1, 0xc000660000, 0xa8f, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068cd80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 484 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe22c001970, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000166cd0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc000166cb0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0027f4a80, 0x96f, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00112c780, 0x2a, 0xc0027f4a80, 0x96f, 0xc0027f4a80, 0x96f, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00112c780, 0x2a, 0xc0027f4a80, 0x96f, 0xaa0378, 0x1, 0xc0027f4a80, 0x96f, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068ce40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 485 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc00078d1e0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:1004 +0x4fc
golang.org/x/net/html.(*parser).parseCurrentToken(0xc00078d1e0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc00078d1e0, 0xc00021bc70, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00229c480, 0x0, 0x0, 0x0, 0xc0022b5200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00229c480, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0022b58f8, 0xc0016c4000, 0x13df, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068cf00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 486 [semacquire]:
bytes.makeSlice(0x81f, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0036f70c0, 0x1, 0x1)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0036f70c0, 0x1189d4e, 0x1, 0x1, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0036f70c0, 0xc000897aa1, 0x56, 0x56, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:440 +0xf7
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0036f70c0, 0xc0039785b0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0036f70c0, 0xc00238ef50, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0036f70c0, 0xc00238ef50, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0036f70c0, 0xc00238ef50, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0036f70c0, 0xc00238ef50, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0036f70c0, 0xc003b438f0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0039030c0, 0xc003b438f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0039030c0, 0xc003b438f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0036f70c0, 0xc003b43810, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0039030c0, 0xc003b43810, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0039030c0, 0xc003b43810, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0036f70c0, 0xc003b43110, 0x1259780, 0x1259790)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0039030c0, 0xc003b43110, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0039030c0, 0xc003b43110, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0036f70c0, 0xc003b43030, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0039030c0, 0xc003b43030, 0x54b170, 0xc003ed05b0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc003b43030, 0x0, 0x0, 0x0, 0x0, 0xc003b43030, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc003219230, 0x0, 0x0, 0x0, 0xd80, 0xc0004ea880, 0xc99, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0004ea880, 0xc99, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc002adba00)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003903518, 0xc00103f620, 0xc001c5e020)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068cfc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 488 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc00090c000, 0xb09, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc00090c000, 0xb09, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc00090c000, 0xb09, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc00090c000, 0xb09, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00090c000, 0xb09, 0x0, 0x0, 0xb09, 0xc00090c000, 0xb09)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00090c000, 0xb09, 0xb09, 0x1000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00322ff80, 0x31, 0xc00090c000, 0xb09, 0xc00090c000, 0xb09, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00322ff80, 0x31, 0xc00090c000, 0xb09, 0xaa0378, 0x1, 0xc00090c000, 0xb09, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068d140, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 489 [runnable]:
github.com/gigawattio/window.Rolling(0xc0055bb300, 0x120, 0x120, 0x3, 0x0, 0x11f, 0xc0055bb300)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0016fb518, 0xc0047e29e0, 0xc0033839e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068d200, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 490 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.(*Model).Predict(0xc00656dae0, 0xc005a1a580, 0x57d, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:153 +0x132
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc005a1a580, 0x57d, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:31 +0xa0
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001272440, 0x36, 0xc005a1a580, 0x57d, 0xc005a1a580, 0x57d, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001272440, 0x36, 0xc005a1a580, 0x57d, 0xaa0378, 0x1, 0xc005a1a580, 0x57d, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068d2c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 491 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005a64760, 0xc0016f92c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0016f93c0, 0xaa05c9, 0x2, 0xc001622000, 0x524, 0x3, 0x118, 0xc0047e2d00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001622000, 0x524, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0016f9518, 0xc00229c780, 0x27, 0xc000dad2c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068d380, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 493 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002269588, 0xaa04f9, 0x2, 0xc00160ec00, 0x38d, 0x1, 0xac, 0xc0034d05e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc002269518, 0xc00229d460, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068d500, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 494 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.(*Model).Predict(0xc005bce480, 0xc001efb500, 0x9c4, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:152 +0xfd
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001efb500, 0x9c4, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:31 +0xa0
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0012bc300, 0x23, 0xc001efb500, 0x9c4, 0xc001efb500, 0x9c4, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0012bc300, 0x23, 0xc001efb500, 0x9c4, 0xaa0378, 0x1, 0xc001efb500, 0x9c4, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068d5c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 495 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005a64980, 0xc002a052c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002a053c0, 0xaa059b, 0x2, 0xc004b3f000, 0x712, 0xc004b3f000, 0xc002a053b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004b3f000, 0x712, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002a05518, 0xc0034d0640, 0x3c, 0xc000e84bc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068d680, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 497 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe15812e480, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0033aadf0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0033aadd0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002f1f000, 0x7f3, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000daced0, 0x2c, 0xc002f1f000, 0x7f3, 0xc002f1f000, 0x7f3, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000daced0, 0x2c, 0xc002f1f000, 0x7f3, 0xaa0378, 0x1, 0xc002f1f000, 0x7f3, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068d800, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 498 [runnable]:
internal/reflectlite.typedmemmove(0xa36900, 0xc0061e59a0, 0xc0061e59b8)
/usr/local/go/src/runtime/mbarrier.go:190 +0x4e
internal/reflectlite.Swapper.func9(0x2dc, 0x2dd)
/usr/local/go/src/internal/reflectlite/swapper.go:71 +0xc1
sort.symMerge_func(0xc0034c37d0, 0xc005312e70, 0x2dc, 0x2dd, 0x2df)
/usr/local/go/src/sort/zfuncversion.go:200 +0xe2
sort.symMerge_func(0xc0034c37d0, 0xc005312e70, 0x2da, 0x2dc, 0x2df)
/usr/local/go/src/sort/zfuncversion.go:247 +0x2ee
sort.symMerge_func(0xc0034c37d0, 0xc005312e70, 0x2da, 0x2dd, 0x2e4)
/usr/local/go/src/sort/zfuncversion.go:244 +0x31d
sort.symMerge_func(0xc0034c37d0, 0xc005312e70, 0x2d0, 0x2d7, 0x2e4)
/usr/local/go/src/sort/zfuncversion.go:247 +0x2ee
sort.symMerge_func(0xc0034c37d0, 0xc005312e70, 0x2d0, 0x2e4, 0x2f8)
/usr/local/go/src/sort/zfuncversion.go:244 +0x31d
sort.stable_func(0xc0034c37d0, 0xc005312e70, 0x405)
/usr/local/go/src/sort/zfuncversion.go:175 +0xc2
sort.SliceStable(0x9de7e0, 0xc0058345a0, 0xc0034c37d0)
/usr/local/go/src/sort/slice.go:27 +0xcb
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc001775000, 0xa81, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:48 +0x1c8
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc001775000, 0xa81, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc001775000, 0xa81, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc001775000, 0xa81, 0x0, 0x0, 0xa81, 0xc001775000, 0xa81)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc001775000, 0xa81, 0xa81, 0xd80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000a14a20, 0x2f, 0xc001775000, 0xa81, 0xc001775000, 0xa81, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000a14a20, 0x2f, 0xc001775000, 0xa81, 0xaa0378, 0x1, 0xc001775000, 0xa81, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068d8c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 499 [semacquire]:
regexp.(*Regexp).replaceAll(0xc000149ae0, 0x0, 0x0, 0x0, 0xc001df6000, 0xbb6, 0x2, 0xc001ad5a58, 0xaa3a2a, 0x4, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc000149ae0, 0xc001df6000, 0xbb6, 0x0, 0x0, 0xc00038e060, 0x127d7a0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
_/home/ec2-user/environment/cluster2/cluster2/base.Sanitize(0xaa3a2a, 0x4, 0xc001df6000, 0xbb6, 0x0, 0x0, 0x10, 0xab52f4)
/home/ec2-user/environment/cluster2/cluster2/base/sanitizer.go:44 +0xc9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068d980, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:304 +0x33a
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 500 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x172af10, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0012299f0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0012299e0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002be9000, 0x79b, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000ccc1c0, 0x37, 0xc002be9000, 0x79b, 0xc002be9000, 0x79b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000ccc1c0, 0x37, 0xc002be9000, 0x79b, 0xaa0378, 0x1, 0xc002be9000, 0x79b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068da40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 501 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc0035ee4e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc0035ee5b0, 0xc0021c4d90, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc000b33200, 0x0, 0x0, 0x0, 0xc0022ad200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc000b33200, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0022ad8f8, 0xc0013de880, 0xcf0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068db00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 502 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc002872f70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc002873040, 0xc00057c000, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0000b38c0, 0x0, 0x0, 0x0, 0xc00286b200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0000b38c0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00286b8f8, 0xc0034a6c00, 0x10eb, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068dbc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 504 [semacquire]:
golang.org/x/net/html.(*Tokenizer).readTag(0xc0035ee680, 0xc002fe8001)
/home/develop/go/src/golang.org/x/net/html/token.go:834 +0x162
golang.org/x/net/html.(*Tokenizer).readStartTag(0xc0035ee680, 0x7fe20000006d)
/home/develop/go/src/golang.org/x/net/html/token.go:779 +0x45
golang.org/x/net/html.(*Tokenizer).Next(0xc0035ee680, 0x1)
/home/develop/go/src/golang.org/x/net/html/token.go:1021 +0x28c
golang.org/x/net/html.(*parser).parse(0xc0035ee750, 0xc0021c5f80, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2284 +0xad
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc000b33220, 0x0, 0x0, 0x0, 0xc003515200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc000b33220, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0035158f8, 0xc000a8c000, 0xadf, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068dd40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 505 [semacquire]:
golang.org/x/net/html.NewTokenizerFragment(0xb7c280, 0xc005a64de0, 0x0, 0x0, 0x20)
/home/develop/go/src/golang.org/x/net/html/token.go:1213 +0x5d
golang.org/x/net/html.NewTokenizer(...)
/home/develop/go/src/golang.org/x/net/html/token.go:1201
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc005a64de0, 0x0, 0x0, 0x0, 0xc00350b200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2337 +0x41
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc005a64de0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00350b8f8, 0xc001aee000, 0x92c, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068de00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 506 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005a64dc0, 0xc001b472c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001b473c0, 0xaa058b, 0x2, 0xc00160f400, 0x390, 0xc00160f400, 0xc001b473b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00160f400, 0x390, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001b47518, 0xc000762be0, 0x37, 0xc0021772c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068dec0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 507 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc005c40000, 0x1ef3, 0xc001b49218, 0xc000611180)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc005c40000, 0x1ef3, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001b493c0, 0xaa04d9, 0x2, 0xc005c40000, 0x1ef3, 0x1, 0x58e, 0xc001001d00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005c40000, 0x1ef3, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001b49518, 0xc005c55d40, 0x2a, 0xc000e7a540)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00068df80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 508 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1fc227b60, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001f9a0b0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001f9a0a0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0014d3800, 0xa8e, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000e84800, 0x37, 0xc0014d3800, 0xa8e, 0xc0014d3800, 0xa8e, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000e84800, 0x37, 0xc0014d3800, 0xa8e, 0xaa0378, 0x1, 0xc0014d3800, 0xa8e, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dc060, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 509 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1b800d4a0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0021ea010, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0021ea000, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001d38d80, 0x479, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000a9e4c0, 0x38, 0xc001d38d80, 0x479, 0xc001d38d80, 0x479, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000a9e4c0, 0x38, 0xc001d38d80, 0x479, 0xaa0378, 0x1, 0xc001d38d80, 0x479, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dc120, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 510 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc001621400, 0x4de, 0x4de, 0xc001d7ae00)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc001621400, 0x4de, 0x7544a2, 0x0)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc005023588, 0xaa04f9, 0x2, 0xc001621400, 0x4de, 0x0, 0xc001d7ae00, 0x7fe260fedc28)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc005023518, 0xc0022275a0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dc1e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 511 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.(*Model).Predict(0xc00656c020, 0xc002bac800, 0x730, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:153 +0x132
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002bac800, 0x730, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:31 +0xa0
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00112cd50, 0x28, 0xc002bac800, 0x730, 0xc002bac800, 0x730, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00112cd50, 0x28, 0xc002bac800, 0x730, 0xaa0378, 0x1, 0xc002bac800, 0x730, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dc2a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 512 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe234009ab0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0028f1f20, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0028f1f10, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002391600, 0x2b0, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000cd5560, 0x22, 0xc002391600, 0x2b0, 0xc002391600, 0x2b0, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000cd5560, 0x22, 0xc002391600, 0x2b0, 0xaa0378, 0x1, 0xc002391600, 0x2b0, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dc360, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 513 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc002c1d000, 0xae1, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc002c1d000, 0xae1, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc002c1d000, 0xae1, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc002c1d000, 0xae1, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc002c1d000, 0xae1, 0x0, 0x0, 0xae1, 0xc002c1d000, 0xae1)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc002c1d000, 0xae1, 0xae1, 0xc00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00322f940, 0x3d, 0xc002c1d000, 0xae1, 0xc002c1d000, 0xae1, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00322f940, 0x3d, 0xc002c1d000, 0xae1, 0xaa0378, 0x1, 0xc002c1d000, 0xae1, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dc420, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 514 [semacquire]:
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0022a72e0, 0x432e00, 0xc0022a7210, 0x9a030e24, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:724 +0x716
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0022a72e0, 0xc003ee5c80, 0x5, 0xc0022a7230, 0x46620f)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0022a72e0, 0xc0022a72f8, 0x43b3b7, 0x1252790, 0xc0022a7200, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xaa64ee, 0x5, 0x7fe238153668, 0xc0022a7301, 0x10100000042ab7a, 0x0, 0xd2f)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xaa64ee, 0x5, 0x0, 0x0, 0xaa0301)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc001098cc0, 0xaa64ee, 0x5, 0xd62)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).getTitleUnmodified(0xc0022a7518, 0xc0034d1280, 0xd2f, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:57 +0x47
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:128 +0x282
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dc4e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 515 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe234002100, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0033303e0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0033303d0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002d5f800, 0x5ca, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00112c990, 0x21, 0xc002d5f800, 0x5ca, 0xc002d5f800, 0x5ca, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00112c990, 0x21, 0xc002d5f800, 0x5ca, 0xaa0378, 0x1, 0xc002d5f800, 0x5ca, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dc5a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 414 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000897ce0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:301 +0x215
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 517 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc000b8f600, 0xc85, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:40 +0x82
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc000b8f600, 0xc85, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc000b8f600, 0xc85, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc000b8f600, 0xc85, 0x0, 0x0, 0xc85, 0xc000b8f600, 0xc85)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc000b8f600, 0xc85, 0xc85, 0x1000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001122ba0, 0x1c, 0xc000b8f600, 0xc85, 0xc000b8f600, 0xc85, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001122ba0, 0x1c, 0xc000b8f600, 0xc85, 0xaa0378, 0x1, 0xc000b8f600, 0xc85, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dc720, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 518 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe234001ce0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002eda160, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002eda150, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002554e00, 0x629, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc003493980, 0x3a, 0xc002554e00, 0x629, 0xc002554e00, 0x629, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc003493980, 0x3a, 0xc002554e00, 0x629, 0xaa0378, 0x1, 0xc002554e00, 0x629, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dc7e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 519 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1f41cf4f0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0028a6180, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0028a6170, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002d0c380, 0x306, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc003493ac0, 0x35, 0xc002d0c380, 0x306, 0xc002d0c380, 0x306, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc003493ac0, 0x35, 0xc002d0c380, 0x306, 0xaa0378, 0x1, 0xc002d0c380, 0x306, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dc8a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 520 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe23020bcd0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0038ee7f0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0038ee7e0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002c74f00, 0x4db, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00112ca80, 0x2f, 0xc002c74f00, 0x4db, 0xc002c74f00, 0x4db, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00112ca80, 0x2f, 0xc002c74f00, 0x4db, 0xaa0378, 0x1, 0xc002c74f00, 0x4db, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dc960, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 521 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe118001280, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0013bc190, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0013bc180, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00315d800, 0x736, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc003493c00, 0x28, 0xc00315d800, 0x736, 0xc00315d800, 0x736, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc003493c00, 0x28, 0xc00315d800, 0x736, 0xaa0378, 0x1, 0xc00315d800, 0x736, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dca20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 522 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe234009eb0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0003ada50, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0003ada40, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002a45800, 0x2e0, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001fce900, 0x4c, 0xc002a45800, 0x2e0, 0xc002a45800, 0x2e0, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001fce900, 0x4c, 0xc002a45800, 0x2e0, 0xaa0378, 0x1, 0xc002a45800, 0x2e0, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dcae0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 524 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe170009970, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0007a1c00, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0007a1bf0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0026eaa80, 0x960, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00226a000, 0x46, 0xc0026eaa80, 0x960, 0xc0026eaa80, 0x960, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00226a000, 0x46, 0xc0026eaa80, 0x960, 0xaa0378, 0x1, 0xc0026eaa80, 0x960, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dcc60, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 396 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0049a3c20, 0x1747eda7c, 0x1251a20, 0xc003213de0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003812c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 526 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe180005740, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002d7b7f0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002d7b7e0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002cdf500, 0x980, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001732a80, 0x38, 0xc002cdf500, 0x980, 0xc002cdf500, 0x980, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001732a80, 0x38, 0xc002cdf500, 0x980, 0xaa0378, 0x1, 0xc002cdf500, 0x980, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dcde0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 529 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003fef588, 0xaa04f9, 0x2, 0xc004574a00, 0x4f6, 0x30, 0xc003fef290, 0x42d308)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0x0, 0xc000ebd6e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:635 +0x82
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc000ebd650, 0xc003fef320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc003fef518, 0xc000ebd530, 0xc001885520)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc003fef518, 0xc000ebd530, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc003fef518, 0xc000ebd530, 0xc000ebd530)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dd020, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 530 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc002844a00, 0x6f5, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc002844a00, 0x6f5, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc002844a00, 0x6f5, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc002844a00, 0x6f5, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc002844a00, 0x6f5, 0x0, 0x0, 0x6f5, 0xc002844a00, 0x6f5)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc002844a00, 0x6f5, 0x6f5, 0x700)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000ccd1c0, 0x3c, 0xc002844a00, 0x6f5, 0xc002844a00, 0x6f5, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000ccd1c0, 0x3c, 0xc002844a00, 0x6f5, 0xaa0378, 0x1, 0xc002844a00, 0x6f5, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dd0e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 531 [semacquire]:
bytes.makeSlice(0x6d, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00422d208, 0x6d, 0x203000)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00422d208, 0xc00234a4d0, 0x6d, 0x1, 0xb856e0, 0xc0059f4460)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0037bbf80)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0037bbf10)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc005f57f20, 0xa9ece0, 0xc005f57ef0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00422d518, 0xc0009c7620, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:384 +0x144
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dd1a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 532 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc0040f3500, 0x938, 0xc004283218, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc0040f3500, 0x938, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0042833c0, 0xaa04fd, 0x2, 0xc0040f3500, 0x938, 0x1, 0x208, 0xc000fefe00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0040f3500, 0x938, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004283518, 0xc001885d60, 0x2f, 0xc0008a05a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dd260, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 533 [runnable]:
regexp.(*Regexp).doExecute(0xc005ad6f00, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0004bcd80, 0xcdb, 0x407, 0x2, ...)
/usr/local/go/src/regexp/exec.go:521 +0x59b
regexp.(*Regexp).replaceAll(0xc005ad6f00, 0x0, 0x0, 0x0, 0xc0004bcd80, 0xcdb, 0x2, 0xc0000ef218, 0xc0004bcd80, 0xcdb, ...)
/usr/local/go/src/regexp/regexp.go:599 +0x1a5
regexp.(*Regexp).ReplaceAllString(0xc005ad6f00, 0xc0004bcd80, 0xcdb, 0xaa0378, 0x1, 0x1, 0x2f)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc004287518, 0xc001885780, 0xc0009c67a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:319 +0x778
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dd320, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 534 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003f1d3c0, 0xaa04fd, 0x2, 0xc002f1b100, 0xd08, 0x1, 0x4a1, 0xc00039fa20)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002f1b100, 0xd08, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003f1d518, 0xc0013976c0, 0x28, 0xc0012bc270)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dd3e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 535 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005048760, 0xc0027891c8, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002789400, 0xaa04f9, 0x2, 0xc0010f80a0, 0x9c, 0xc002789240, 0x42d308, 0x8)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords.func1(0x12, 0xc00552c240)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:174 +0x86
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc00102e5a0, 0xc0027892c8, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords(0xc002789388)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:173 +0xb0
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc002789388, 0xc0004d05d0, 0xaa04f9, 0x2, 0xc002789518, 0x8, 0x9ecc80, 0xc002789301, 0xc000623cc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:43 +0xf5
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc002789518, 0xc0004d05d0, 0xaa04f9, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dd4a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 536 [semacquire]:
strings.Replace(0xc00259e600, 0x1888, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0x13, 0xc00259e600, 0x1888)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0032058f8, 0xc000884c80, 0x1878, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dd560, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 538 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00522cc00, 0xc0015a11b0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0015a1588, 0xaa04f9, 0x2, 0xc0062fc000, 0x9bf, 0xc00427bc01, 0xc0040b83f0, 0xc0015a1230)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc0015a1518, 0xc001b5d470, 0xc001b5d710)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:528 +0xfd
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0015a1518, 0xc0023018c0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dd6e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 539 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc001289dc0, 0xdc, 0xdc, 0xc001d7ae00)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc001289dc0, 0xdc, 0x7544a2, 0x0)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003aa7588, 0xaa04f9, 0x2, 0xc001289dc0, 0xdc, 0x9, 0x1b, 0xc005834280)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc003aa7518, 0xc0009c7ac0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dd7a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 540 [semacquire]:
strings.genSplit(0xc002430000, 0xf78, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc004ed7000, 0x2c7, 0x2c7)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000ca53c0, 0xaa0599, 0x2, 0xc0033ce000, 0xf78, 0x1, 0x2c7, 0xc00000fbc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0033ce000, 0xf78, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000ca5518, 0xc0033095c0, 0x39, 0xc002dded00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dd860, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 542 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc003efd0c0, 0xc002ae7950, 0x8f, 0x8f, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:430 +0x233
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc003efd0c0, 0xc001081c70, 0xc003efc1b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc003efd0c0, 0xc001081c00, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc003efd0c0, 0xc001081c00, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc003efd0c0, 0xc001081c00, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc003efd0c0, 0xc001081c00, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc003efd0c0, 0xc0010815e0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc003a070c0, 0xc0010815e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc003a070c0, 0xc0010815e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc003efd0c0, 0xc001081500, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc003a070c0, 0xc001081500, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc003a070c0, 0xc001081500, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc003efd0c0, 0xc005f442a0, 0xc003be6380, 0x7fe260fedc28)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc003a070c0, 0xc005f442a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc003a070c0, 0xc005f442a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc003efd0c0, 0xc005f441c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc003a070c0, 0xc005f441c0, 0x54b170, 0xc00151cdd0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc005f441c0, 0x0, 0x0, 0x0, 0x0, 0xc005f441c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0057653b0, 0x0, 0x0, 0x0, 0xc00, 0xc002140000, 0xb19, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc002140000, 0xb19, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc000380c60)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003a07518, 0xc0033099a0, 0xc0024093e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dd9e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 543 [semacquire]:
strings.genSplit(0xc0065a4c80, 0xc2a, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x53, 0xc0065a4000, 0x7fe23806a5f8)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002df73c0, 0xaa058b, 0x2, 0xc0065a4000, 0xc2a, 0xc0065a4000, 0xc002df73b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0065a4000, 0xc2a, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002df7518, 0xc00000e960, 0x35, 0xc002ddf200)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ddaa0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 544 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc004488000, 0x88b, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:87 +0x379
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc004488000, 0x88b, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc004488000, 0x88b, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc004488000, 0x88b, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc004488000, 0x88b, 0x0, 0x0, 0x88b, 0xc004488000, 0x88b)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc004488000, 0x88b, 0x88b, 0xa80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00456c230, 0x43, 0xc004488000, 0x88b, 0xc004488000, 0x88b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00456c230, 0x43, 0xc004488000, 0x88b, 0xaa0378, 0x1, 0xc004488000, 0x88b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ddb60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 545 [runnable]:
strings.genSplit(0xc00605c000, 0x1dbf, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc002cc1500, 0x618, 0x618)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001f2f3c0, 0xaa05c9, 0x2, 0xc005ada000, 0x1dbf, 0x6, 0x618, 0xc001cfc4e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005ada000, 0x1dbf, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001f2f518, 0xc001c5ea80, 0x38, 0xc000aa85c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ddc20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 546 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/jaytaylor/html2text.FromHTMLNode(0xc005fca150, 0x0, 0x0, 0x0, 0x0, 0xc005fca150, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:81 +0x10d
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc004744810, 0x0, 0x0, 0x0, 0xc00, 0xc003bb3000, 0xa85, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc003bb3000, 0xa85, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc003493ec0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000e35518, 0xc004c383a0, 0xc004c399a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ddce0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 548 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe23400ad90, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0027c48f0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0027c48d0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00337e100, 0x50d, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc003264040, 0x3e, 0xc00337e100, 0x50d, 0xc00337e100, 0x50d, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc003264040, 0x3e, 0xc00337e100, 0x50d, 0xaa0378, 0x1, 0xc00337e100, 0x50d, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006dde60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 549 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0054af0c0, 0xc002f83050, 0x8a, 0x8a, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:430 +0x233
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0054af0c0, 0xc0011248c0, 0xc0054ae1b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0054af0c0, 0xc001124850, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0054af0c0, 0xc001124850, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0054af0c0, 0xc001124850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0054af0c0, 0xc001124850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0054af0c0, 0xc0053595e0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00305d0c0, 0xc0053595e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00305d0c0, 0xc0053595e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0054af0c0, 0xc005359500, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00305d0c0, 0xc005359500, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00305d0c0, 0xc005359500, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0054af0c0, 0xc002a9ddc0, 0xc001e94000, 0x7fe260fed560)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00305d0c0, 0xc002a9ddc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00305d0c0, 0xc002a9ddc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0054af0c0, 0xc002a9dce0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00305d0c0, 0xc002a9dce0, 0x54b170, 0xc002eedd40)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc002a9dce0, 0x0, 0x0, 0x0, 0x0, 0xc002a9dce0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc000cfed80, 0x0, 0x0, 0x0, 0x1800, 0xc002e24800, 0x162e, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc002e24800, 0x162e, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5714b91100000000)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00305d518, 0xc0032a9980, 0xc004cda400)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ddf20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 550 [semacquire]:
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc0020a3620, 0x59, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:560 +0x135
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0052d10c0, 0xc0029f2620, 0xc0052d01b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052d10c0, 0xc0029f25b0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0052d10c0, 0xc0029f25b0, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0052d10c0, 0xc0029f25b0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0052d10c0, 0xc0029f25b0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052d10c0, 0xc0026e2a10, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00064f0c0, 0xc0026e2a10, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00064f0c0, 0xc0026e2a10, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052d10c0, 0xc0026e28c0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00064f0c0, 0xc0026e28c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00064f0c0, 0xc0026e28c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052d10c0, 0xc00391d650, 0xc00064ed00, 0x444010)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00064f0c0, 0xc00391d650, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00064f0c0, 0xc00391d650, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052d10c0, 0xc00391d570, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00064f0c0, 0xc00391d570, 0x54b170, 0xc0026fa750)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc00391d570, 0x0, 0x0, 0x0, 0x0, 0xc00391d570, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc00201c1e0, 0x0, 0x0, 0x0, 0x1800, 0xc000559800, 0x17d4, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc000559800, 0x17d4, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc00270c800)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00064f518, 0xc00522df20, 0xc0032a8a00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ee000, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 551 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc0022b0ea0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc0022b12b0, 0xc005598070, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc001246f80, 0x0, 0x0, 0x0, 0xc004133200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc001246f80, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0041338f8, 0xc001b6d300, 0x1167, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ee0c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 552 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0019693c0, 0xaa0569, 0x2, 0xc000530d00, 0x84f, 0x2f, 0x1a4, 0xc000c45ea0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000530d00, 0x84f, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001969518, 0xc0009c69c0, 0x2b, 0xc0008a0420)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ee180, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 553 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe140002080, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0021de7c0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0021de7a0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000bf7b00, 0x824, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0031e0100, 0x32, 0xc000bf7b00, 0x824, 0xc000bf7b00, 0x824, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0031e0100, 0x32, 0xc000bf7b00, 0x824, 0xaa0378, 0x1, 0xc000bf7b00, 0x824, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ee240, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 555 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2340a1c20, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0008c8170, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0008c8130, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0033f0000, 0x770, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000ccc480, 0x3b, 0xc0033f0000, 0x770, 0xc0033f0000, 0x770, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000ccc480, 0x3b, 0xc0033f0000, 0x770, 0xaa0378, 0x1, 0xc0033f0000, 0x770, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ee3c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 557 [semacquire]:
strings.genSplit(0xc005f81000, 0xf23, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x42fa20, 0x337, 0x337)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0045193c0, 0xaa04fd, 0x2, 0xc005902000, 0xf23, 0x1, 0x337, 0xc005c540a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005902000, 0xf23, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004519518, 0xc002f70c00, 0x3c, 0xc003c3c100)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ee540, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 559 [semacquire]:
strings.genSplit(0xc00582e400, 0x855, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc00682e000, 0x1c6, 0x1c6)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00209d3c0, 0xaa04d9, 0x2, 0xc004e21200, 0x855, 0x1, 0x1c6, 0xc0023002e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004e21200, 0x855, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00209d518, 0xc003212840, 0x32, 0xc003265c40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ee6c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 560 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.ToLower(0xc006449000, 0xddb, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:600 +0x222
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0033633c0, 0xaa0569, 0x2, 0xc006449000, 0xddb, 0x57, 0x321, 0xc002f70b20)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc006449000, 0xddb, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003363518, 0xc002409e40, 0x29, 0xc000dacff0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ee780, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 561 [semacquire]:
regexp.(*bitState).reset(0xc004644000, 0xc0000acdb0, 0xe74, 0x2)
/usr/local/go/src/regexp/backtrack.go:100 +0x18b
regexp.(*Regexp).backtrack(0xc0001fb400, 0x0, 0x0, 0x0, 0xc000972000, 0xe74, 0xb49, 0x2, 0xc0036ba620, 0x0, ...)
/usr/local/go/src/regexp/backtrack.go:319 +0x116
regexp.(*Regexp).doExecute(0xc0001fb400, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc000972000, 0xe74, 0xb49, 0x2, ...)
/usr/local/go/src/regexp/exec.go:535 +0x453
regexp.(*Regexp).replaceAll(0xc0001fb400, 0x0, 0x0, 0x0, 0xc000972000, 0xe74, 0x2, 0xc003b47a58, 0xaa6444, 0x5, ...)
/usr/local/go/src/regexp/regexp.go:599 +0x1a5
regexp.(*Regexp).ReplaceAllString(0xc0001fb400, 0xc000972000, 0xe74, 0xaa0378, 0x1, 0xc000909000, 0xe74)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
_/home/ec2-user/environment/cluster2/cluster2/base.Sanitize(0xaa6444, 0x5, 0xc000972000, 0xe74, 0xaa0378, 0x1, 0xc000972000, 0xe74)
/home/ec2-user/environment/cluster2/cluster2/base/sanitizer.go:44 +0xc9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ee840, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:305 +0x37c
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 563 [runnable]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003f1f3c0, 0xaa059b, 0x2, 0xc001c31500, 0x1347, 0xc001c31500, 0xc003f1f3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:81 +0x1a0
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001c31500, 0x1347, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003f1f518, 0xc0024080a0, 0x32, 0xc001733380)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ee9c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 564 [semacquire]:
strings.Replace(0xc000d65300, 0x1038, 0xaa60d9, 0x5, 0xaa1fd2, 0x3, 0x2, 0xc000d65300, 0x1038)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0022bb518, 0xc004c39840, 0xc0009c6240)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:316 +0x67b
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006eea80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 565 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2340a0e20, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0038d20b0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0038d20a0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000da8d00, 0x824, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0025fe040, 0x32, 0xc000da8d00, 0x824, 0xc000da8d00, 0x824, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0025fe040, 0x32, 0xc000da8d00, 0x824, 0xaa0378, 0x1, 0xc000da8d00, 0x824, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006eeb40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 566 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc002c8e660, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001b67518, 0xc0000b3020, 0x27, 0xc00112d980)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006eec00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 567 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002a6d3c0, 0xaa04f9, 0x2, 0xc003310000, 0x5a1, 0xc003310000, 0xc002a6d3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003310000, 0x5a1, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002a6d518, 0xc0034d1460, 0x37, 0xc002dde815)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006eecc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 568 [semacquire]:
bytes.makeSlice(0x904, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0031b9388, 0xcc, 0xd0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0031b9388, 0xc00198c5b0, 0xcc, 0x5165d7, 0xc005808c5f, 0xc0010d8440)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00044a230)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00044a1c0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000449500)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000449420)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0028ba150)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc000ccad80, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0031b9518, 0xc0000b37c0, 0x34, 0xc0046af200)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006eed80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 569 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc000109790, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc000109860, 0xc0006b3ab0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00000eca0, 0x0, 0x0, 0x0, 0xc002a5d200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00000eca0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc002a5d8f8, 0xc002298800, 0x15df, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006eee40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 570 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.ToLower(0xc004f3c000, 0x782, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:600 +0x222
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0031bb3c0, 0xaa059b, 0x2, 0xc004f3c000, 0x782, 0x18, 0x1a3, 0xc000c44380)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004f3c000, 0x782, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0031bb518, 0xc002f71a80, 0x35, 0xc001732740)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006eef00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 571 [semacquire]:
strings.genSplit(0xc004d85400, 0xb7f, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0055cea80, 0x284, 0x284)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003c533c0, 0xaa058b, 0x2, 0xc004d81800, 0xb7f, 0x7, 0x284, 0xc00642a500)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004d81800, 0xb7f, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003c53518, 0xc001cfc880, 0x26, 0xc000c53c20)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006eefc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 572 [semacquire]:
strconv.formatBits(0x0, 0x0, 0x0, 0xf2, 0xa, 0xc0008c0000, 0x753f20, 0xab29a3, 0xc, 0xc0001da230, ...)
/usr/local/go/src/strconv/itoa.go:200 +0x318
strconv.FormatInt(0xf2, 0xa, 0xf4, 0x0)
/usr/local/go/src/strconv/itoa.go:29 +0xdb
strconv.Itoa(...)
/usr/local/go/src/strconv/itoa.go:35
github.com/advancedlogic/GoOse.(*ContentExtractor).updateScore(0xc0008c1518, 0xc002b705d0, 0xfffffffffffffffe)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:493 +0x6f
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0008c1518, 0xc0000b22c0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:435 +0x67f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ef080, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 573 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe230221ee0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00188e3f0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00188e390, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002413500, 0x1332, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00112c7b0, 0x30, 0xc002413500, 0x1332, 0xc002413500, 0x1332, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00112c7b0, 0x30, 0xc002413500, 0x1332, 0xaa0378, 0x1, 0xc002413500, 0x1332, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ef140, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 575 [semacquire]:
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0031b72e0, 0x432e00, 0xc0031b7210, 0x9a030e24, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:724 +0x716
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0031b72e0, 0xc0017d9b00, 0x5, 0xc0031b7230, 0x46620f)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0031b72e0, 0xc0031b72f8, 0x43b3b7, 0x1252790, 0xc0031b7200, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xaa64ee, 0x5, 0x7fe23812ca20, 0xc0031b7301, 0x10100000042ab7a, 0x0, 0x1107)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xaa64ee, 0x5, 0x0, 0x0, 0xaa0301)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0005acc00, 0xaa64ee, 0x5, 0x17a9)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).getTitleUnmodified(0xc0031b7518, 0xc0000b3520, 0x1107, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:57 +0x47
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:128 +0x282
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ef2c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 576 [semacquire]:
github.com/PuerkitoBio/goquery.newSingleSelection(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:113
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc0042e74a0, 0xc0028031d8, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0xb5
github.com/advancedlogic/GoOse.(*ContentExtractor).nodesToCheck(0xc002803518, 0xc001001cc0, 0x28, 0xc0042e7440, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:551 +0x183
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc002803518, 0xc001001cc0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:374 +0x50
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ef380, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 577 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1f0476890, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc004a02af0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc004a02ae0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001a4c400, 0x82b, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002ddec80, 0x34, 0xc001a4c400, 0x82b, 0xc001a4c400, 0x82b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002ddec80, 0x34, 0xc001a4c400, 0x82b, 0xaa0378, 0x1, 0xc001a4c400, 0x82b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ef440, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 578 [semacquire]:
bytes.makeSlice(0x79, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0025f1198, 0x79, 0x42ca96)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0025f1198, 0xc000df7f00, 0x79, 0x28, 0x28, 0xc0033cc5d0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc005641500)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc005641490)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc003366180, 0xc0033cc5d0, 0xc0025f1210)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc0025f1518, 0xc003366180, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:527 +0xc5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0025f1518, 0xc001dd2700, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ef500, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 579 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc005468680, 0x567, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:71 +0x129
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc005468680, 0x567, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc005468680, 0x567, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc005468680, 0x567, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc005468680, 0x567, 0x0, 0x0, 0x567, 0xc005468680, 0x567)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc005468680, 0x567, 0x567, 0x580)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0026fdba0, 0x12, 0xc005468680, 0x567, 0xc005468680, 0x567, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0026fdba0, 0x12, 0xc005468680, 0x567, 0xaa0378, 0x1, 0xc005468680, 0x567, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ef5c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 580 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0053e5588, 0xaa04f9, 0x2, 0xc005a7a120, 0x8c, 0x5, 0xc0053e5290, 0x42d308)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0x6, 0xc00135b860)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:635 +0x82
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc005b1f380, 0xc0053e5320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc0053e5518, 0xc005b1e810, 0xc0063e6d00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc0053e5518, 0xc005b1e810, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc0053e5518, 0xc005b1e810, 0xc005b1e810)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ef680, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 581 [runnable]:
strings.Index(0xc004f3d000, 0x76b, 0xaa0378, 0x1, 0x12a)
/usr/local/go/src/strings/strings.go:1028 +0x67a
strings.genSplit(0xc004f3d000, 0x76b, 0xaa0378, 0x1, 0x0, 0x12a, 0x76b, 0xc00388d800, 0x77d)
/usr/local/go/src/strings/strings.go:260 +0xf3
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0020eb518, 0xc0000b3500, 0xc004909e80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:321 +0x7b8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ef740, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 582 [semacquire]:
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc005a7b290, 0x90, 0x2, 0xc006152198, 0xc00000000a, 0x2, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc005a7b290, 0x90, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0061530c0, 0xc0015f65b0, 0xc0061521b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0061530c0, 0xc0005efb20, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0061530c0, 0xc0005efb20, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0061530c0, 0xc0005efb20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0061530c0, 0xc0005efb20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0061530c0, 0xc001cbc1c0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0043cd0c0, 0xc001cbc1c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0043cd0c0, 0xc001cbc1c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0061530c0, 0xc001cbc0e0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0043cd0c0, 0xc001cbc0e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0043cd0c0, 0xc001cbc0e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0061530c0, 0xc002655c70, 0x7fe23810e308, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0043cd0c0, 0xc002655c70, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0043cd0c0, 0xc002655c70, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0061530c0, 0xc002655b90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0043cd0c0, 0xc002655b90, 0x54b170, 0xc000386a90)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc002655b90, 0x0, 0x0, 0x0, 0x0, 0xc002655b90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0029fd7d0, 0x0, 0x0, 0x0, 0xd80, 0xc001dda000, 0xd63, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc001dda000, 0xd63, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x54222db700000000)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0043cd518, 0xc0007633c0, 0xc000b32740)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ef800, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 583 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0030ca300, 0x6ba, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:59 +0x21d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0030ca300, 0x6ba, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0030ca300, 0x6ba, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc0030ca300)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0030ca300, 0x6ba, 0x0, 0x0, 0x6ba, 0x700, 0xc0030ca300)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0030ca300, 0x6ba, 0x6ba, 0x700)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000764140, 0x43, 0xc0030ca300, 0x6ba, 0xc0030ca300, 0x6ba, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000764140, 0x43, 0xc0030ca300, 0x6ba, 0xaa0378, 0x1, 0xc0030ca300, 0x6ba, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ef8c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 584 [semacquire]:
strings.genSplit(0xc005210400, 0xb52, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0055a4000, 0x262, 0x262)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0038833c0, 0xaa05c9, 0x2, 0xc002e94000, 0xb52, 0x3, 0x262, 0xc0033087a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002e94000, 0xb52, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003883518, 0xc000762ba0, 0x39, 0xc0023e3b80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006ef980, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 585 [runnable]:
unicode.SimpleFold(0x4e, 0xc003c68fa0)
/usr/local/go/src/unicode/letter.go:336 +0x14f
regexp/syntax.(*compiler).rune(0xc006832b30, 0xc002b6a308, 0x1, 0xe, 0xd5, 0x600000003)
/usr/local/go/src/regexp/syntax/compile.go:271 +0x1df
regexp/syntax.(*compiler).compile(0xc006832b30, 0xc001912af0, 0x200000001)
/usr/local/go/src/regexp/syntax/compile.go:110 +0x12c
regexp/syntax.(*compiler).compile(0xc006832b30, 0xc001912b60, 0x0)
/usr/local/go/src/regexp/syntax/compile.go:156 +0xb29
regexp/syntax.Compile(0xc001912b60, 0xc001912b60, 0x1, 0x1)
/usr/local/go/src/regexp/syntax/compile.go:83 +0x10a
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:178 +0xd7
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc006832ff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc006832ff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc0032fffac)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc006832ff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc006832ff8, 0x1, 0xc0021fcde0, 0xc0103c36301, 0x280460fee2f0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc006832ff8, 0x4, 0xd95, 0xc002244201, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc002244000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203001, 0xc003bd1a00, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc005d58f00, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0017f78f8, 0xc003a88c00, 0x11da, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006efa40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 586 [semacquire]:
strings.Replace(0xc0041fa000, 0x164e, 0xaa582d, 0x5, 0xaa7dce, 0x6, 0x1, 0xc0041fa000, 0x164e)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:178 +0x153
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0042858f8, 0xc00072b800, 0x164d, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006efb00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 587 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc000762380, 0xc00220d220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00220d588, 0xaa04f9, 0x2, 0xc004aa0280, 0x95, 0xd, 0x1a, 0xc001cfc520)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00220d518, 0xc0032a8020, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006efbc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 588 [semacquire]:
bytes.makeSlice(0xe3f, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0031e9388, 0x65, 0x7d)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0031e9388, 0xc001ae4620, 0x65, 0x5165d7, 0xc0054e029c, 0xc00076e148)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc003ff97a0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc003ff9730)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00055fce0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00055fc00)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00055f500)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc001b63380, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0031e9518, 0xc000c45c40, 0x29, 0xc0006e65d0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006efc80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 589 [semacquire]:
regexp.makeOnePass(0xc00135b5f0, 0xc00135b5f0)
/usr/local/go/src/regexp/onepass.go:444 +0x310
regexp.compileOnePass(0xc00135b590, 0xc00135b590)
/usr/local/go/src/regexp/onepass.go:500 +0x129
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc002864ff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc002864ff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc003331f14)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc002864ff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc002864ff8, 0x1, 0xc005625cb0, 0xc01009e5101, 0x280460fec108)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc002864ff8, 0x4, 0xd0f, 0xc002a3b201, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc002a3b000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x42c59c, 0x7fe23811ef80, 0x33f)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc00135b560, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0028658f8, 0xc001654c00, 0x1298, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006efd40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 590 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc004680000, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0020e9518, 0xc00115b4c0, 0x23, 0xc000a15b00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006efe00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 39 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc003dc3c00, 0x60f, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc003dc3c00, 0x60f, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc003dc3c00, 0x60f, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc003dc3c00, 0x60f, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc003dc3c00, 0x60f, 0x0, 0x0, 0x60f, 0xc003dc3c00, 0x60f)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc003dc3c00, 0x60f, 0x60f, 0x800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002dde840, 0x38, 0xc003dc3c00, 0x60f, 0xc003dc3c00, 0x60f, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002dde840, 0x38, 0xc003dc3c00, 0x60f, 0xaa0378, 0x1, 0xc003dc3c00, 0x60f, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0006eff80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 40 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0044c2000, 0x6ba, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:71 +0x129
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0044c2000, 0x6ba, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0044c2000, 0x6ba, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0044c2000, 0x6ba, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc0044c2000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0044c2000, 0x6ba, 0x0, 0x0, 0x6ba, 0x700, 0xc0044c2000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0044c2000, 0x6ba, 0x6ba, 0x700)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000ea6eb0, 0x43, 0xc0044c2000, 0x6ba, 0xc0044c2000, 0x6ba, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000ea6eb0, 0x43, 0xc0044c2000, 0x6ba, 0xaa0378, 0x1, 0xc0044c2000, 0x6ba, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427da0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 41 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0023d2000, 0x75b, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x2e6
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0023d2000, 0x75b, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0023d2000, 0x75b, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0023d2000, 0x75b, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0023d2000, 0x75b, 0x0, 0x0, 0x75b, 0xc0023d2000, 0x75b)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0023d2000, 0x75b, 0x75b, 0x900)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000e7aa50, 0x2c, 0xc0023d2000, 0x75b, 0xc0023d2000, 0x75b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000e7aa50, 0x2c, 0xc0023d2000, 0x75b, 0xaa0378, 0x1, 0xc0023d2000, 0x75b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427f20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 42 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0039bd000, 0xda2, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x2e6
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0039bd000, 0xda2, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0039bd000, 0xda2, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0039bd000, 0xda2, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0039bd000, 0xda2, 0x0, 0x0, 0xda2, 0xc0039bd000, 0xda2)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0039bd000, 0xda2, 0xda2, 0x1000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0026dc870, 0x23, 0xc0039bd000, 0xda2, 0xc0039bd000, 0xda2, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0026dc870, 0x23, 0xc0039bd000, 0xda2, 0xaa0378, 0x1, 0xc0039bd000, 0xda2, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8060, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 43 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc00185f000, 0xda2, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:87 +0x379
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc00185f000, 0xda2, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc00185f000, 0xda2, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc00185f000, 0xda2, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00185f000, 0xda2, 0x0, 0x0, 0xda2, 0xc00185f000, 0xda2)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00185f000, 0xda2, 0xda2, 0x1000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002eea011, 0x35, 0xc00185f000, 0xda2, 0xc00185f000, 0xda2, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002eea011, 0x35, 0xc00185f000, 0xda2, 0xaa0378, 0x1, 0xc00185f000, 0xda2, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8120, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 44 [semacquire]:
regexp/syntax.Parse(0xaa5b70, 0x5, 0xc0005e00d4, 0x7fe260fef001, 0x47f290, 0xc005b31e30)
/usr/local/go/src/regexp/syntax/parse.go:712 +0x54
regexp.compile(0xaa5b70, 0x5, 0xd4, 0xc000039338, 0xc0005e3248, 0x509d68)
/usr/local/go/src/regexp/regexp.go:170 +0x5a
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
regexp.MustCompile(0xaa5b70, 0x5, 0xaa0378)
/usr/local/go/src/regexp/regexp.go:309 +0x4b
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0005e3518, 0xc0032138e0, 0xc0000b2ec0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:324 +0x7f8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f81e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 45 [semacquire]:
golang.org/x/net/html/atom.String(0xc0021f6145, 0x8, 0xebb, 0xebb, 0xc0021f614f)
/home/develop/go/src/golang.org/x/net/html/atom/atom.go:77 +0xc6
golang.org/x/net/html.(*Tokenizer).Token(0xc0000980d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x12a
golang.org/x/net/html.(*parser).parse(0xc0052f2270, 0xc006402070, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0007631c0, 0x0, 0x0, 0x0, 0xc0020f7200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0007631c0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0020f78f8, 0xc000d97500, 0x9d0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f82a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 46 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc001dbcf00, 0x485, 0xc002121218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc001dbcf00, 0x485, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0021213c0, 0xaa04df, 0x2, 0xc001dbcf00, 0x485, 0x1, 0x141, 0xc005a65280)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001dbcf00, 0x485, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002121518, 0xc003383b60, 0x79, 0xc001137c80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8360, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 48 [semacquire]:
bytes.makeSlice(0x5ef, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc002f06e10, 0x5, 0x3)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc002f06e10, 0xc0038ff633, 0x5, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc002f06e10, 0xc0038ff633, 0x5, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc002f06e10, 0xc00663bab0, 0x1, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc002f06e10, 0xc00663ba40, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc002f06e10, 0xc0041d6a10, 0xb84d20, 0xc002f06e10)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc002f06e10, 0xc0041d6a10, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc004c85080, 0x1952a84d8, 0x1251a20, 0xc00137ed20, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f84e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 49 [runnable]:
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc00642b140, 0xc00078b110, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:127 +0x159
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc00642b140, 0xc00078afc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:133 +0x9e
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc00642b140, 0xc00078a9a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:133 +0x9e
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc00642b140, 0xc00078a8c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:133 +0x9e
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc00642b140, 0xc00078a1c0, 0x0, 0x0, 0x0, 0x10, 0x9ed160, 0x46a5ec)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:133 +0x9e
github.com/andybalholm/cascadia.Selector.MatchAll(0xc00642b140, 0xc00078a1c0, 0xc00355f1e8, 0x747325, 0xc00004d180)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:124 +0x46
github.com/PuerkitoBio/goquery.findWithMatcher.func1(0x0, 0xc00078a0e0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:547 +0x8e
github.com/PuerkitoBio/goquery.mapNodes(0xc000900f50, 0x1, 0x1, 0xc00355f348, 0x1, 0x0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:693 +0xf6
github.com/PuerkitoBio/goquery.findWithMatcher(0xc000900f50, 0x1, 0x1, 0xb856e0, 0xc00642b140, 0xc00355f3c8, 0x46e6ce, 0xc002cb3000)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:543 +0x81
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc005bb27b0, 0xaa64ee, 0x5, 0xc7c)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x85
github.com/advancedlogic/GoOse.(*ContentExtractor).getTitleUnmodified(0xc00355f518, 0xc00642aec0, 0x761, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:57 +0x47
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:128 +0x282
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f85a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 594 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc000108ea0, 0xc003350208, 0x8)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc000108ea0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc000108ea0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc000108ea0, 0xc000448c40, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc003309c60, 0x0, 0x0, 0x0, 0xc004591200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc003309c60, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0045918f8, 0xc002348000, 0x1ff7, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8660, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 595 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc0010b0cc0, 0xc0035612c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0035613c0, 0xaa04fd, 0x2, 0xc00231c000, 0x1473, 0x1, 0x3fd, 0xc0010b0ca0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00231c000, 0x1473, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003561518, 0xc0010b0880, 0x58, 0xc0010dc600)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8720, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 597 [semacquire]:
strings.genSplit(0xc005f30000, 0x11b5, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x42f9b8, 0x363, 0x363)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0038413c0, 0xaa04fd, 0x2, 0xc004998600, 0x11b5, 0x5c, 0x363, 0xc003113ca0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004998600, 0x11b5, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003841518, 0xc0010b0420, 0x5e, 0xc00271eea0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f88a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 598 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc001a3a000, 0x117c, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:87 +0x379
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc001a3a000, 0x117c, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc001a3a000, 0x117c, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc001a3a000, 0x117c, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc001a3a000, 0x117c, 0x0, 0x0, 0x117c, 0xc001a3a000, 0x117c)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc001a3a000, 0x117c, 0x117c, 0x1300)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000b5ebd0, 0x4b, 0xc001a3a000, 0x117c, 0xc001a3a000, 0x117c, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000b5ebd0, 0x4b, 0xc001a3a000, 0x117c, 0xaa0378, 0x1, 0xc001a3a000, 0x117c, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8960, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 600 [semacquire]:
bytes.makeSlice(0x1a2, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00205f208, 0x1a2, 0x203000)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00205f208, 0xc000cdafc0, 0x1a2, 0x1, 0xb856e0, 0xc00656ce00)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc002053a40)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0020539d0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc00226c9c0, 0xa9ece0, 0xc00226c990)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00205f518, 0xc0004e6640, 0x6b)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:384 +0x144
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8ae0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 601 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc001c4f000, 0xe78, 0xc002029218, 0xc00004d500)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc001c4f000, 0xe78, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0020293c0, 0xaa04d9, 0x2, 0xc001c4f000, 0xe78, 0x1, 0x2c5, 0xc00115b6e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001c4f000, 0xe78, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002029518, 0xc0004e66e0, 0x2a, 0xc000cd50b0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8ba0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 602 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc0028735f0, 0xc0045ffea0, 0xdd)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc0028735f0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0028735f0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0028735f0, 0xc00065efc0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0000b3e40, 0x0, 0x0, 0x0, 0xc000437200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0000b3e40, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0004378f8, 0xc002157800, 0x27b2, 0xc0009d78e1, 0x4, 0xc0009d78e6)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8c60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 603 [semacquire]:
regexp/syntax.(*compiler).inst(...)
/usr/local/go/src/regexp/syntax/compile.go:173
regexp/syntax.(*compiler).rune(0xc00450eb30, 0xc000b73d98, 0x1, 0xa, 0xd5, 0xe00000007)
/usr/local/go/src/regexp/syntax/compile.go:267 +0x27c
regexp/syntax.(*compiler).compile(0xc00450eb30, 0xc005706a10, 0x200000001)
/usr/local/go/src/regexp/syntax/compile.go:110 +0x12c
regexp/syntax.(*compiler).compile(0xc00450eb30, 0xc005706a80, 0x0)
/usr/local/go/src/regexp/syntax/compile.go:156 +0xb29
regexp/syntax.Compile(0xc005706a80, 0xc005706a80, 0x1, 0x1)
/usr/local/go/src/regexp/syntax/compile.go:83 +0x10a
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:178 +0xd7
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc00450eff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc00450eff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc0023d41ec)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc00450eff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc00450eff8, 0x1, 0xc001d2e950, 0xc0102c4a700, 0x280460fece98)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc00450eff8, 0x4, 0x33e, 0xc001404c01, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc001404000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc002873930, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0038d78c0, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00450f8f8, 0xc000a9a000, 0x1c86, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8d20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 604 [semacquire]:
regexp.makeOnePass.func1(0xc, 0xc0023d4300, 0x10, 0x10, 0x1)
/usr/local/go/src/regexp/onepass.go:381 +0xad2
regexp.makeOnePass(0xc0038d7980, 0xc0038d7980)
/usr/local/go/src/regexp/onepass.go:448 +0x398
regexp.compileOnePass(0xc0038d7920, 0xc0038d7920)
/usr/local/go/src/regexp/onepass.go:500 +0x129
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0040e0ff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0040e0ff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc0023d42cc)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0040e0ff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0040e0ff8, 0x1, 0xc001d2e970, 0xc01033de001, 0x280460fec108)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0040e0ff8, 0x4, 0x852, 0xc000973701, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc000973000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc002873ad0, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0038d78f0, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0040e18f8, 0xc002278000, 0x1778, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8de0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 605 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc001ed3920, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00171f518, 0xc0010007a0, 0x36, 0xc000aa9800)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8ea0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 606 [semacquire]:
strings.genSplit(0xc004efdb00, 0x871, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc002e3a000, 0x1c6, 0x1c6)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0046113c0, 0xaa0569, 0x2, 0xc004218000, 0x871, 0x1, 0x1c6, 0xc0010008e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004218000, 0x871, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004611518, 0xc003308940, 0x3b, 0xc000a9e400)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8f60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 607 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc003a15000, 0x158b, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:87 +0x379
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc003a15000, 0x158b, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc003a15000, 0x158b, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc003a15000, 0x158b, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc003a15000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc003a15000, 0x158b, 0x0, 0x0, 0x158b, 0x1800, 0xc003a15000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc003a15000, 0x158b, 0x158b, 0x1800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000a14750, 0x21, 0xc003a15000, 0x158b, 0xc003a15000, 0x158b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000a14750, 0x21, 0xc003a15000, 0x158b, 0xaa0378, 0x1, 0xc003a15000, 0x158b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9020, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 608 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.ToLower(0xc00385d000, 0xe93, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:600 +0x222
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0037b13c0, 0xaa04f9, 0x2, 0xc00385d000, 0xe93, 0x5, 0x2dc, 0xc004c381a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00385d000, 0xe93, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0037b1518, 0xc0009c7700, 0x2e, 0xc000b6d890)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f90e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 609 [semacquire]:
github.com/PuerkitoBio/goquery.appendWithoutDuplicates(0xc0025d4f00, 0x10, 0x10, 0xc0024a3f00, 0x1c, 0x20, 0xc0018b7020, 0xc003956e78, 0x1, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/utilities.go:137 +0x1cf
github.com/PuerkitoBio/goquery.mapNodes(0xc003254c00, 0x1, 0x1, 0xc0018b70f0, 0x7fe260fece98, 0x0, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:694 +0x159
github.com/PuerkitoBio/goquery.getSiblingNodes(0xc003254c00, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc003956e78, ...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:598 +0x101
github.com/PuerkitoBio/goquery.(*Selection).Siblings(0xc00477e9f0, 0xc00568a630)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:268 +0x59
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc0018b7518, 0xc00477e9f0, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:517 +0x20f
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0018b7518, 0xc0024883e0, 0x2c)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f91a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 611 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0042e6a20, 0x19342c66d, 0x1251a20, 0xc001c5ee80, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9320, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 612 [semacquire]:
strings.genSplit(0xc004f6a800, 0x741, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc004d40000, 0x1c2, 0x1c2)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0027a73c0, 0xaa05c9, 0x2, 0xc0062d2000, 0x741, 0x4, 0x1c2, 0xc00103fde0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0062d2000, 0x741, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0027a7518, 0xc004c38640, 0x39, 0xc004559c80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f93e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 613 [runnable]:
bytes.makeSlice.func1()
/usr/local/go/src/bytes/buffer.go:224 +0x68
bytes.makeSlice(0x4774, 0xc005789000, 0x4774, 0x4774)
/usr/local/go/src/bytes/buffer.go:229 +0x96
bytes.(*Buffer).grow(0xc001b23208, 0x224, 0xab)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc001b23208, 0xc000ab1d40, 0x224, 0x5165d7, 0xc000016614, 0xc001795a28)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc002135f10)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc002135ea0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00212e770)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc004b1ec60, 0xc004b1f3b0, 0xc0008a3800)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc001b23518, 0xc001dd2720, 0x3a)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:384 +0x144
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f94a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 614 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc003afea80, 0x9ea, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:81 +0x208
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc003afea80, 0x9ea, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc003afea80, 0x9ea, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc003afea80, 0x9ea, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc003afea80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc003afea80, 0x9ea, 0x0, 0x0, 0x9ea, 0xa80, 0xc003afea80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc003afea80, 0x9ea, 0x9ea, 0xa80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000a9f7c0, 0x3d, 0xc003afea80, 0x9ea, 0xc003afea80, 0x9ea, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000a9f7c0, 0x3d, 0xc003afea80, 0x9ea, 0xaa0378, 0x1, 0xc003afea80, 0x9ea, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9560, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 615 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.ToLower(0xc00239a800, 0x721, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:600 +0x222
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00350f3c0, 0xaa04df, 0x2, 0xc00239a800, 0x721, 0x1, 0x180, 0xc0059f4540)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00239a800, 0x721, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00350f518, 0xc00115b7e0, 0x43, 0xc00189a550)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9620, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 616 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc003139000, 0xec6, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:71 +0x129
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc003139000, 0xec6, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc003139000, 0xec6, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc003139000, 0xec6, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc003139000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc003139000, 0xec6, 0x0, 0x0, 0xec6, 0x1000, 0xc003139000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc003139000, 0xec6, 0xec6, 0x1000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000dacc90, 0x27, 0xc003139000, 0xec6, 0xc003139000, 0xec6, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000dacc90, 0x27, 0xc003139000, 0xec6, 0xaa0378, 0x1, 0xc003139000, 0xec6, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f96e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 617 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc001ecaa00, 0x135d, 0x5165d7, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc001ecaa00, 0x135d, 0xc000dffc1c, 0x4)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0031cf3c0, 0xaa0569, 0x2, 0xc001ecaa00, 0x135d, 0xc001ecaa00, 0xc0031cf3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001ecaa00, 0x135d, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0031cf518, 0xc005a64380, 0x32, 0xc002292480)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f97a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 618 [runnable]:
strings.Index(0xc000cfac30, 0xd0, 0xaa0378, 0x1, 0x1c)
/usr/local/go/src/strings/strings.go:1028 +0x67a
strings.genSplit(0xc000cfac30, 0xd0, 0xaa0378, 0x1, 0x0, 0x1c, 0xd0, 0xc000cfa9c0, 0xd0)
/usr/local/go/src/strings/strings.go:260 +0xf3
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003259588, 0xaa04f9, 0x2, 0xc000cfa9c0, 0xd0, 0x7, 0xc003259290, 0x42d308)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0x4, 0xc0062217a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:635 +0x82
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc006221530, 0xc003259320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc003259518, 0xc006220300, 0xc0022264e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc003259518, 0xc006220300, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc003259518, 0xc006220300, 0xc006220300)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9860, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 619 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1d820ffb0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00128a710, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00128a700, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001aa5800, 0x6dd, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000efe780, 0x3a, 0xc001aa5800, 0x6dd, 0xc001aa5800, 0x6dd, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000efe780, 0x3a, 0xc001aa5800, 0x6dd, 0xaa0378, 0x1, 0xc001aa5800, 0x6dd, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9920, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 620 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc002832000, 0xdd7, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:48 +0x172
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc002832000, 0xdd7, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc002832000, 0xdd7, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc002832000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc002832000, 0xdd7, 0x0, 0x0, 0xdd7, 0x1000, 0xc002832000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc002832000, 0xdd7, 0xdd7, 0x1000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000efec00, 0x2b, 0xc002832000, 0xdd7, 0xc002832000, 0xdd7, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000efec00, 0x2b, 0xc002832000, 0xdd7, 0xaa0378, 0x1, 0xc002832000, 0xdd7, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f99e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 621 [semacquire]:
strings.Replace(0xc00621c000, 0xfa3, 0xaa03a3, 0x1, 0xaa0378, 0x1, 0x32, 0xc00621c000, 0xfa3)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002a5f518, 0xc0004e6f60, 0xc0004e78e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:318 +0x708
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9aa0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 622 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc002ff9000, 0xd9c, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc002ff9000, 0xd9c, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc002ff9000, 0xd9c, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc002ff9000, 0xd9c, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc002ff9000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc002ff9000, 0xd9c, 0x0, 0x0, 0xd9c, 0x1300, 0xc002ff9000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc002ff9000, 0xd9c, 0xd9c, 0x1300)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000eff800, 0x32, 0xc002ff9000, 0xd9c, 0xc002ff9000, 0xd9c, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000eff800, 0x32, 0xc002ff9000, 0xd9c, 0xaa0378, 0x1, 0xc002ff9000, 0xd9c, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9b60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 623 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005a64900, 0xc001f352c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001f353c0, 0xaa04f9, 0x2, 0xc004b8e600, 0x1278, 0xc004b8e600, 0xc001f353b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004b8e600, 0x1278, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001f35518, 0xc0022277c0, 0x39, 0xc000aa9300)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9c20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 624 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc002ffab60, 0xc003ce82e0, 0x8)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc002ffab60, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc002ffab60)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc002ffab60, 0xc00130ca10, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc004cdb720, 0x0, 0x0, 0x0, 0xc001039200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc004cdb720, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0010398f8, 0xc0012f0000, 0x38f8, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9ce0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 625 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0029f7500, 0x14db, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x2e6
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0029f7500, 0x14db, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0029f7500, 0x14db, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0029f7500, 0x14db, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0029f7500, 0x14db, 0x0, 0x0, 0x14db, 0xc0029f7500, 0x14db)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0029f7500, 0x14db, 0x14db, 0x1500)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001287f40, 0x3a, 0xc0029f7500, 0x14db, 0xc0029f7500, 0x14db, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001287f40, 0x3a, 0xc0029f7500, 0x14db, 0xaa0378, 0x1, 0xc0029f7500, 0x14db, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9da0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 627 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe21020c360, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0035ed240, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0035ed230, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000bf7200, 0x807, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001286800, 0x37, 0xc000bf7200, 0x807, 0xc000bf7200, 0x807, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001286800, 0x37, 0xc000bf7200, 0x807, 0xaa0378, 0x1, 0xc000bf7200, 0x807, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9f20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 628 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0050b0a80, 0x94e, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0050b0a80, 0x94e, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0050b0a80, 0x94e, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0050b0a80, 0x94e, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc0050b0a80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0050b0a80, 0x94e, 0x0, 0x0, 0x94e, 0xa80, 0xc0050b0a80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0050b0a80, 0x94e, 0x94e, 0xa80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001286a99, 0x26, 0xc0050b0a80, 0x94e, 0xc0050b0a80, 0x94e, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001286a99, 0x26, 0xc0050b0a80, 0x94e, 0xaa0378, 0x1, 0xc0050b0a80, 0x94e, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834000, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 629 [semacquire]:
regexp.onePassCopy(0xc00037c240, 0xc00037c240)
/usr/local/go/src/regexp/onepass.go:225 +0x5d
regexp.compileOnePass(0xc00037c240, 0xc00037c240)
/usr/local/go/src/regexp/onepass.go:497 +0x11b
regexp.compile(0xabc10c, 0x11, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0005b9300, 0x0, 0xabc106, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0005b9300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0005b9300, 0x0, 0xb856e0, 0xc00039e188, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0005b9300, 0xc0039a4ee0, 0x0, 0x8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0005b9300, 0x42ca96, 0xc00037c210, 0x30, 0x28, 0xa4b3a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xabc101, 0x1d, 0xc0005b9370, 0x42d308, 0x30, 0xa4b3a0, 0xb85601)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xabc101, 0x1d, 0xc0032a8940, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc002558cf0, 0xabc101, 0x1d, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc0005b9518, 0xc00115aee0, 0xabc101, 0x1d, 0x6a, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:133 +0x3a7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018340c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 630 [semacquire]:
strings.genSplit(0xc002c4f000, 0xf50, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc000b3b000, 0x379, 0x379)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001a1b3c0, 0xaa04fd, 0x2, 0xc00550e000, 0xf50, 0x1, 0x379, 0xc00103ede0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00550e000, 0xf50, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001a1b518, 0xc0009c66c0, 0x60, 0xc002716072)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834180, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 631 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0005df400, 0xaa04f9, 0x2, 0xc00128c480, 0x211, 0xc0005df240, 0x42d308, 0x8)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords.func1(0xa, 0xc00419f9b0)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:174 +0x86
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc001b4cc90, 0xc0005df2c8, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords(0xc0005df388)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:173 +0xb0
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc0005df388, 0xc004d86f00, 0xaa04f9, 0x2, 0xc0005df518, 0x8, 0x9ecc80, 0xc0005df301, 0xc000900fa0)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:43 +0xf5
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc0005df518, 0xc004d86f00, 0xaa04f9, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834240, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 632 [semacquire]:
regexp.(*Regexp).expand(0xc000149400, 0xc005a5dda0, 0x20, 0x20, 0xaa0378, 0x1, 0x0, 0x0, 0x0, 0xc000a34a90, ...)
/usr/local/go/src/regexp/regexp.go:948 +0xa9b
regexp.(*Regexp).ReplaceAllString.func1(0xc005a5dda0, 0x20, 0x20, 0xc00403e310, 0x2, 0x2, 0xc000a34a90, 0xc7, 0x1f)
/usr/local/go/src/regexp/regexp.go:558 +0xbf
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc000a34a90, 0xc7, 0x2, 0xc003592198, 0xc00000000a, 0x2, ...)
/usr/local/go/src/regexp/regexp.go:616 +0x2e9
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc000a34a90, 0xc7, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0035930c0, 0xc0047e51f0, 0xc0035921b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0035930c0, 0xc0047e5180, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0035930c0, 0xc0047e5180, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0035930c0, 0xc0047e5180, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0035930c0, 0xc0047e5180, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0035930c0, 0xc002a35ea0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0038330c0, 0xc002a35ea0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0038330c0, 0xc002a35ea0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0035930c0, 0xc002a35dc0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0038330c0, 0xc002a35dc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0038330c0, 0xc002a35dc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0035930c0, 0xc002a356c0, 0x20300100000000, 0x7fe10134dfff)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0038330c0, 0xc002a356c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0038330c0, 0xc002a356c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0035930c0, 0xc002a355e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0038330c0, 0xc002a355e0, 0x54b170, 0xc002873ba0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc002a355e0, 0x0, 0x0, 0x0, 0x0, 0xc002a355e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0030a8e40, 0x0, 0x0, 0x0, 0x2000, 0xc004798000, 0x1c41, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc004798000, 0x1c41, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc0005cd440)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003833518, 0xc0009c6860, 0xc00522d580)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 633 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0040f2000, 0x95e, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:81 +0x208
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0040f2000, 0x95e, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0040f2000, 0x95e, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0040f2000, 0x95e, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc0040f2000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0040f2000, 0x95e, 0x0, 0x0, 0x95e, 0xc00, 0xc0040f2000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0040f2000, 0x95e, 0x95e, 0xc00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00464287d, 0x37, 0xc0040f2000, 0x95e, 0xc0040f2000, 0x95e, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00464287d, 0x37, 0xc0040f2000, 0x95e, 0xaa0378, 0x1, 0xc0040f2000, 0x95e, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018343c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 634 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).breakLongLines(0xc00551f0c0, 0xc0025e4bb0, 0xae, 0x2, 0x2, 0x6)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:462 +0x58c
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc00551f0c0, 0xc0025e4bb0, 0xae, 0xae, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:426 +0x63
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00551f0c0, 0xc000d58cb0, 0xc00551e1b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00551f0c0, 0xc000d58c40, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc00551f0c0, 0xc000d58c40, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00551f0c0, 0xc000d58c40, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00551f0c0, 0xc000d58c40, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00551f0c0, 0xc00297fe30, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00425b0c0, 0xc00297fe30, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00425b0c0, 0xc00297fe30, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00551f0c0, 0xc00297fd50, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00425b0c0, 0xc00297fd50, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00425b0c0, 0xc00297fd50, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00551f0c0, 0xc003231b90, 0x7fe2380c75e0, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00425b0c0, 0xc003231b90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00425b0c0, 0xc003231b90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00551f0c0, 0xc003231ab0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00425b0c0, 0xc003231ab0, 0x54b170, 0xc00078c4e0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc003231ab0, 0x0, 0x0, 0x0, 0x0, 0xc003231ab0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc000ccaf90, 0x0, 0x0, 0x0, 0x1300, 0xc0024fd900, 0x1039, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0024fd900, 0x1039, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc0024e4d80)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00425b518, 0xc001c5f4a0, 0xc001c5fd40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 635 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0031e53c0, 0xaa059b, 0x2, 0xc00083e800, 0xb88, 0x1a, 0x28f, 0xc00000f360)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00083e800, 0xb88, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0031e5518, 0xc00000ede0, 0x40, 0xc001273b0a)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834540, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 636 [runnable]:
bytes.makeSlice(0x40f, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc005fb10c0, 0x1, 0x1)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc005fb10c0, 0x1189d54, 0x1, 0x1, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc005fb10c0, 0xc000f63d10, 0x4e, 0x4e, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:440 +0xf7
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc005fb10c0, 0xc003911810, 0xc005fb01b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005fb10c0, 0xc0039117a0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc005fb10c0, 0xc0039117a0, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc005fb10c0, 0xc0039117a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc005fb10c0, 0xc0039117a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005fb10c0, 0xc0039b2930, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00451b0c0, 0xc0039b2930, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00451b0c0, 0xc0039b2930, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005fb10c0, 0xc0039b2850, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00451b0c0, 0xc0039b2850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00451b0c0, 0xc0039b2850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005fb10c0, 0xc0049f6850, 0x7fe23811b578, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00451b0c0, 0xc0049f6850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00451b0c0, 0xc0049f6850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005fb10c0, 0xc0049f6770, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00451b0c0, 0xc0049f6770, 0x54b170, 0xc000387e10)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc0049f6770, 0x0, 0x0, 0x0, 0x0, 0xc0049f6770, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc002d603f0, 0x0, 0x0, 0x0, 0x1800, 0xc000ec8000, 0x1551, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc000ec8000, 0x1551, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000cccd00)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00451b518, 0xc0000b3880, 0xc000b33300)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 637 [semacquire]:
regexp/syntax.(*compiler).inst(...)
/usr/local/go/src/regexp/syntax/compile.go:173
regexp/syntax.(*compiler).rune(0xc0020f2e38, 0xc004f8f0c8, 0x1, 0x6, 0xd5, 0x600000003)
/usr/local/go/src/regexp/syntax/compile.go:267 +0x27c
regexp/syntax.(*compiler).compile(0xc0020f2e38, 0xc005f0a620, 0x200000001)
/usr/local/go/src/regexp/syntax/compile.go:110 +0x12c
regexp/syntax.(*compiler).compile(0xc0020f2e38, 0xc005f0a690, 0x0)
/usr/local/go/src/regexp/syntax/compile.go:156 +0xb29
regexp/syntax.Compile(0xc005f0a690, 0xc005f0a690, 0x1, 0x1)
/usr/local/go/src/regexp/syntax/compile.go:83 +0x10a
regexp.compile(0xaba6cc, 0xe, 0xd4, 0x10, 0xa2acc0, 0x1cb)
/usr/local/go/src/regexp/regexp.go:178 +0xd7
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0020f3300, 0x0, 0xaba6c6, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0020f3300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0020f3300, 0x0, 0x0, 0x0, 0xb856e0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0020f3300, 0x75a856, 0x0, 0xc002c77c70, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0020f3300, 0x79, 0xc0020f3350, 0x42cc84, 0xc005bb36b0, 0x30)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xaba6c1, 0x1a, 0x1, 0xc0020f3368, 0xc0020f3380, 0x42d308, 0x30)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xaba6c1, 0x1a, 0x1, 0xb856e0, 0xc005bcefa0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc002fb7710, 0xaba6c1, 0x1a, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc0020f3518, 0xc0034d13c0, 0xaba6c1, 0x1a, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:134 +0x401
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018346c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 638 [semacquire]:
strings.genSplit(0xc003287600, 0x523, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc004a2ec00, 0x116, 0x116)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0020f53c0, 0xaa0569, 0x2, 0xc00038ac00, 0x523, 0x2f, 0x116, 0xc000762280)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00038ac00, 0x523, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0020f5518, 0xc0032128c0, 0x42, 0xc000f631d0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834780, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 639 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe190001230, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0031de130, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0031de120, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00308ae00, 0x6e7, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000907d40, 0x2e, 0xc00308ae00, 0x6e7, 0xc00308ae00, 0x6e7, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000907d40, 0x2e, 0xc00308ae00, 0x6e7, 0xaa0378, 0x1, 0xc00308ae00, 0x6e7, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834840, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 640 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0009b4000, 0xea2, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x2e6
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0009b4000, 0xea2, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0009b4000, 0xea2, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0009b4000, 0xea2, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0009b4000, 0xea2, 0x0, 0x0, 0xea2, 0xc0009b4000, 0xea2)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0009b4000, 0xea2, 0xea2, 0x1000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000114a50, 0x2c, 0xc0009b4000, 0xea2, 0xc0009b4000, 0xea2, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000114a50, 0x2c, 0xc0009b4000, 0xea2, 0xaa0378, 0x1, 0xc0009b4000, 0xea2, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834900, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 641 [runnable]:
strings.Index(0xc001109400, 0x24c, 0xaa0378, 0x1, 0x5a)
/usr/local/go/src/strings/strings.go:1028 +0x67a
strings.genSplit(0xc001109400, 0x24c, 0xaa0378, 0x1, 0x0, 0x5a, 0x2a6, 0xc001108c80, 0x24c)
/usr/local/go/src/strings/strings.go:260 +0xf3
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0032f7588, 0xaa04f9, 0x2, 0xc001108c80, 0x24c, 0x19, 0x43, 0xc000fefca0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0032f7518, 0xc002408860, 0x4c)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018349c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 642 [semacquire]:
golang.org/x/net/html.NewTokenizerFragment(0xb7c280, 0xc005a64ca0, 0x0, 0x0, 0x20)
/home/develop/go/src/golang.org/x/net/html/token.go:1213 +0x5d
golang.org/x/net/html.NewTokenizer(...)
/home/develop/go/src/golang.org/x/net/html/token.go:1201
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc005a64ca0, 0x0, 0x0, 0x0, 0xc0034b5200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2337 +0x41
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc005a64ca0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0034b58f8, 0xc003a88c00, 0x1046, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834a80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 643 [semacquire]:
bytes.makeSlice(0x67d, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc001b621e0, 0x4f, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc001b621e0, 0xc001419ea0, 0x4f, 0x6, 0xffffffffffffffff, 0x1)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc001b621e0, 0xc001419ea0, 0x4f, 0x74e006, 0xc0035a0040)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc001b621e0, 0xc00076c070, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc001b621e0, 0xc00076c000, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001b621e0, 0xc00076bf10, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001b621e0, 0xc00076be30, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001b621e0, 0xc00076b730, 0xb84d20, 0xc001b621e0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc001b621e0, 0xc00076b730, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc002c4d530, 0x193af1d86, 0x1251a20, 0xc001cfd260, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834b40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 644 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00380f3c0, 0xaa0503, 0x2, 0xc000b1a000, 0x1cfa, 0x4, 0x67d, 0xc00522c960)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000b1a000, 0x1cfa, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00380f518, 0xc00522c5a0, 0x56, 0xc0022f2c8c)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834c00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 645 [semacquire]:
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc0008a11d0, 0x29, 0x2, 0xc00422fe30, 0x60, 0xa81260, ...)
/usr/local/go/src/regexp/regexp.go:597 +0x57
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc0008a11d0, 0x29, 0xaa0378, 0x1, 0xc00422feb8, 0x46620f)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0042310c0, 0xc003928070, 0xc0017b4cc0, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0042310c0, 0xc003928000, 0x1259780, 0x1259790)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0042310c0, 0xc003928000, 0x4c, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0042310c0, 0xc003928000, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0042310c0, 0xc00134d1f0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0042310c0, 0xc00134d1f0, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0042310c0, 0xc00134d1f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0042310c0, 0xc00134d1f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0042310c0, 0xc00392bab0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0041b30c0, 0xc00392bab0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0041b30c0, 0xc00392bab0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0042310c0, 0xc00392b9d0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0041b30c0, 0xc00392b9d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0041b30c0, 0xc00392b9d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0042310c0, 0xc00392b2d0, 0xc001d7aa80, 0x7fe260fee9b8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0041b30c0, 0xc00392b2d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0041b30c0, 0xc00392b2d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0042310c0, 0xc00392b180, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0041b30c0, 0xc00392b180, 0x54b170, 0xc000098270)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc00392b180, 0x0, 0x0, 0x0, 0x0, 0xc00392b180, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc00518af90, 0x0, 0x0, 0x0, 0xc00, 0xc00083e800, 0xabe, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc00083e800, 0xabe, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc0008a0de0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0041b3518, 0xc0027ee1a0, 0xc0027eea60)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834cc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 646 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc0035ee820, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x260
golang.org/x/net/html.(*parser).parse(0xc0035ee8f0, 0xc00226e690, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc000b33240, 0x0, 0x0, 0x0, 0xc001c4b200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc000b33240, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc001c4b8f8, 0xc001639400, 0x22b5, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834d80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 647 [semacquire]:
golang.org/x/net/html.NewTokenizerFragment(0xb7c280, 0xc005a64e40, 0x0, 0x0, 0x20)
/home/develop/go/src/golang.org/x/net/html/token.go:1213 +0x5d
golang.org/x/net/html.NewTokenizer(...)
/home/develop/go/src/golang.org/x/net/html/token.go:1201
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc005a64e40, 0x0, 0x0, 0x0, 0xc003511200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2337 +0x41
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc005a64e40, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0035118f8, 0xc001b8e000, 0xee7, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834e40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 648 [semacquire]:
strings.genSplit(0xc005ae5000, 0xb32, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x42fa20, 0x24f, 0x24f)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0008953c0, 0xaa04d9, 0x2, 0xc000bd6800, 0xb32, 0x1, 0x24f, 0xc005965c60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000bd6800, 0xb32, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000895518, 0xc005049c80, 0x36, 0xc0032656c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834f00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 649 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x172a330, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00076ee30, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00076ee20, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00319f500, 0xa6c, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0012bcd20, 0x26, 0xc00319f500, 0xa6c, 0xc00319f500, 0xa6c, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0012bcd20, 0x26, 0xc00319f500, 0xa6c, 0xaa0378, 0x1, 0xc00319f500, 0xa6c, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001834fc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 650 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe234004360, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001bbf010, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001bbf000, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0016f0300, 0x658, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000aa9e40, 0x3a, 0xc0016f0300, 0x658, 0xc0016f0300, 0x658, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000aa9e40, 0x3a, 0xc0016f0300, 0x658, 0xaa0378, 0x1, 0xc0016f0300, 0x658, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835080, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 651 [runnable]:
github.com/fatih/set.(*Set).Add(0xc00000e9e0, 0xc001bd5220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001bd5588, 0xaa04f9, 0x2, 0xc00307ec60, 0x101, 0x1, 0x1f, 0xc00000e8e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc001bd5518, 0xc0010b0d00, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835140, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 653 [semacquire]:
bytes.makeSlice(0xc4, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc000337138, 0x44, 0x40)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc000337138, 0xc000ea6550, 0x44, 0x5165d7, 0xc00535dbc0, 0xc0013c03a5)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc002386a80)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc002386a10)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc002386930)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc000e517d0, 0xaa03ab, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).isHighLinkDensity(0xc000337518, 0xc000e517d0, 0x2)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:566 +0x95
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc000337518, 0xc0010b1980, 0x46)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:386 +0x1ae
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018352c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 654 [runnable]:
github.com/PuerkitoBio/goquery.getChildrenWithSiblingType(0xc00229f110, 0x0, 0xc00229fc70, 0x0, 0xc0035cc528, 0x1, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:667 +0x18f
github.com/PuerkitoBio/goquery.getSiblingNodes.func2(0x0, 0xc00229fc70, 0x1, 0xc0035cc528, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:599 +0x45
github.com/PuerkitoBio/goquery.mapNodes(0xc0035cc300, 0x1, 0x1, 0xc003cbb0f0, 0x7fe260fef080, 0x0, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:693 +0xf6
github.com/PuerkitoBio/goquery.getSiblingNodes(0xc0035cc300, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0035cc530, ...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:598 +0x101
github.com/PuerkitoBio/goquery.(*Selection).Siblings(0xc000ebc6c0, 0xc002830cc0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:268 +0x59
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc003cbb518, 0xc000ebc6c0, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:517 +0x20f
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc003cbb518, 0xc003383cc0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835380, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 655 [semacquire]:
golang.org/x/net/html/atom.String(0xc0054340fa, 0x8, 0xf06, 0xf06, 0xc005434104)
/home/develop/go/src/golang.org/x/net/html/atom/atom.go:77 +0xc6
golang.org/x/net/html.(*Tokenizer).Token(0xc00452f380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x12a
golang.org/x/net/html.(*parser).parse(0xc00452f520, 0xc0049f63f0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc003213240, 0x0, 0x0, 0x0, 0xc00355d200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc003213240, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00355d8f8, 0xc00105d900, 0x10bd, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835440, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 656 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc0004f9300, 0x197a, 0xc004263218, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc0004f9300, 0x197a, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0042633c0, 0xaa059b, 0x2, 0xc0004f9300, 0x197a, 0x1e, 0x5b4, 0xc002227b60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0004f9300, 0x197a, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004263518, 0xc002227780, 0x41, 0xc004e86b40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835500, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 657 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc001884ce0, 0xc00213d2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00213d3c0, 0xaa04fd, 0x2, 0xc000839000, 0x74d, 0x1, 0x1c8, 0xc0009c63c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000839000, 0x74d, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00213d518, 0xc002301c20, 0x2a, 0xc000cd5bf0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018355c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 658 [runnable]:
github.com/fatih/set.(*Set).Has(0xc00039e1c0, 0xc0042652c0, 0x1, 0x1, 0xc000480100)
/home/develop/go/src/github.com/fatih/set/set_ts.go:71 +0x13
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0042653c0, 0xaa04df, 0x2, 0xc0043eb500, 0x149f, 0x1, 0x44e, 0xc0010008a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:81 +0x1e7
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0043eb500, 0x149f, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004265518, 0xc002408360, 0x30, 0xc000a14000)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835680, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 659 [semacquire]:
strings.genSplit(0xc0020d4000, 0xec3, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0061ef000, 0x35f, 0x35f)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00318d3c0, 0xaa0503, 0x2, 0xc005675000, 0xec3, 0x4, 0x35f, 0xc005965440)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005675000, 0xec3, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00318d518, 0xc0059650a0, 0x3c, 0xc002b6b8c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835740, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 660 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc002f544e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc002f54750, 0xc0055312d0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0009c6040, 0x0, 0x0, 0x0, 0xc0014c1200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0009c6040, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0014c18f8, 0xc001a74000, 0xff2, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835800, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 661 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe234001b80, 0x7fe138003110)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000aea0b0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc000aea0a0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00147b800, 0x17c8, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000b732c0, 0x38, 0xc00147b800, 0x17c8, 0xc00147b800, 0x17c8, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000b732c0, 0x38, 0xc00147b800, 0x17c8, 0xaa0378, 0x1, 0xc00147b800, 0x17c8, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018358c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 662 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inHeadIM(0xc00266f1e0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:640 +0x2e8
golang.org/x/net/html.(*parser).parseCurrentToken(0xc00266f1e0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc00266f1e0, 0xc006803030, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc000fef8c0, 0x0, 0x0, 0x0, 0xc003b59200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc000fef8c0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc003b598f8, 0xc0032a0f00, 0x23f8, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835980, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 663 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc002d61bf0, 0x17187cb35, 0x1251a20, 0xc00522ce40, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835a40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 664 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1780026b0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002daedc0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002daedb0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0003c2000, 0x1b63, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0033540c0, 0x34, 0xc0003c2000, 0x1b63, 0xc0003c2000, 0x1b63, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0033540c0, 0x34, 0xc0003c2000, 0x1b63, 0xaa0378, 0x1, 0xc0003c2000, 0x1b63, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835b00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 665 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2241c6410, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00162b930, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00162b920, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0026af000, 0xf68, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0020554c0, 0x1d, 0xc0026af000, 0xf68, 0xc0026af000, 0xf68, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0020554c0, 0x1d, 0xc0026af000, 0xf68, 0xaa0378, 0x1, 0xc0026af000, 0xf68, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835bc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 666 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc000098a90, 0xc0023de000, 0x212)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc000098a90, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc000098a90)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc000098a90, 0xc004cbfb20, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc000763240, 0x0, 0x0, 0x0, 0xc000e15200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc000763240, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc000e158f8, 0xc00300b900, 0x23f9, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835c80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 667 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc00078cdd0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc00078cea0, 0xc00021b960, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00229c440, 0x0, 0x0, 0x0, 0xc001497200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00229c440, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0014978f8, 0xc002ea4000, 0x1c5f, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835d40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 668 [runnable]:
unicode.IsSpace(0xc00000000a, 0xc001680e60)
/usr/local/go/src/unicode/graphic.go:126 +0x98
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0069df0c0, 0xc001680e60, 0x9a, 0x7, 0xc001680e60)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:431 +0x25d
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0069df0c0, 0xc004ca8540, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:176 +0xb28
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0069df0c0, 0xc004ca8540, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069df0c0, 0xc004ca8460, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000f250c0, 0xc004ca8460, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000f250c0, 0xc004ca8460, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069df0c0, 0xc004ca8380, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000f250c0, 0xc004ca8380, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000f250c0, 0xc004ca8380, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069df0c0, 0xc005573ea0, 0xc004303180, 0x7fe260fee9b8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000f250c0, 0xc005573ea0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000f250c0, 0xc005573ea0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069df0c0, 0xc005573dc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000f250c0, 0xc005573dc0, 0x54b170, 0xc002eec9c0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc005573dc0, 0x0, 0x0, 0x0, 0x0, 0xc005573dc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc00465cae0, 0x0, 0x0, 0x0, 0xd80, 0xc001e81b00, 0xd6c, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc001e81b00, 0xd6c, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc00305f920)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000f25518, 0xc0007620a0, 0xc001884160)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835e00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 669 [semacquire]:
encoding/json.(*decodeState).literalStore(0xc00227f810, 0xc005962dfb, 0x11, 0x105, 0x9e4c80, 0xc004a06688, 0x18d, 0x0, 0x126, 0xc00227f838)
/usr/local/go/src/encoding/json/decode.go:988 +0x1007
encoding/json.(*decodeState).value(0xc00227f810, 0x9e4c80, 0xc004a06688, 0x18d, 0x9e4c80, 0xc004a06688)
/usr/local/go/src/encoding/json/decode.go:401 +0x1de
encoding/json.(*decodeState).object(0xc00227f810, 0xa49de0, 0xc004a06680, 0x199, 0xab20e1, 0xb)
/usr/local/go/src/encoding/json/decode.go:782 +0x12e5
encoding/json.(*decodeState).value(0xc00227f810, 0xa49de0, 0xc004a06680, 0x199, 0xa49de0, 0xc004a06680)
/usr/local/go/src/encoding/json/decode.go:387 +0x6d
encoding/json.(*decodeState).array(0xc00227f810, 0x9cfa40, 0xc005ca6640, 0x16, 0xc00227f838, 0x5b)
/usr/local/go/src/encoding/json/decode.go:575 +0x1a7
encoding/json.(*decodeState).value(0xc00227f810, 0x9cfa40, 0xc005ca6640, 0x16, 0xc0030418a8, 0x532927)
/usr/local/go/src/encoding/json/decode.go:377 +0xfd
encoding/json.(*decodeState).unmarshal(0xc00227f810, 0x9cfa40, 0xc005ca6640, 0xc00227f838, 0x0)
/usr/local/go/src/encoding/json/decode.go:180 +0x1f0
encoding/json.Unmarshal(0xc005962dc0, 0x126, 0x140, 0x9cfa40, 0xc005ca6640, 0x140, 0xaa366e)
/usr/local/go/src/encoding/json/decode.go:107 +0x112
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.(*Model).Predict(0xc005ca6620, 0xc0011e3b00, 0x878, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:153 +0x16a
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0011e3b00, 0x878, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:31 +0xa0
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001418230, 0x42, 0xc0011e3b00, 0x878, 0xc0011e3b00, 0x878, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001418230, 0x42, 0xc0011e3b00, 0x878, 0xaa0378, 0x1, 0xc0011e3b00, 0x878, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835ec0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 670 [semacquire]:
golang.org/x/net/html.NewTokenizerFragment(0xb7c280, 0xc005a64c20, 0x0, 0x0, 0x20)
/home/develop/go/src/golang.org/x/net/html/token.go:1213 +0x5d
golang.org/x/net/html.NewTokenizer(...)
/home/develop/go/src/golang.org/x/net/html/token.go:1201
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc005a64c20, 0x0, 0x0, 0x0, 0xc003103200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2337 +0x41
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc005a64c20, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0031038f8, 0xc003076000, 0xf75, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001835f80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 671 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe21c000f20, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000351290, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc000351270, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001621900, 0x500, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00260620e, 0x67, 0xc001621900, 0x500, 0xc001621900, 0x500, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00260620e, 0x67, 0xc001621900, 0x500, 0xaa0378, 0x1, 0xc001621900, 0x500, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4060, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 672 [semacquire]:
bytes.makeSlice(0x1e00, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0008770e8, 0x200, 0x800)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).ReadFrom(0xc0008770e8, 0xb7b620, 0xc005dcc5a0, 0x203001, 0x203001, 0x203001)
/usr/local/go/src/bytes/buffer.go:202 +0x48
io/ioutil.readAll(0xb7b620, 0xc005dcc5a0, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:36 +0xe3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc005dcc5a0, 0xc0008771d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc005dcc5a0, 0x0, 0x0, 0x0, 0x1300, 0xc006422c00, 0x1112, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc006422c00, 0x1112, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc000c5f340)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000877518, 0xc0047e2720, 0xc004909920)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4120, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 673 [semacquire]:
regexp.makeOnePass.func1(0x4, 0xc0015cca40, 0x10, 0x10, 0x1)
/usr/local/go/src/regexp/onepass.go:390 +0x9d8
regexp.makeOnePass(0xc0025580c0, 0xc0025580c0)
/usr/local/go/src/regexp/onepass.go:448 +0x398
regexp.compileOnePass(0xc002558060, 0xc002558060)
/usr/local/go/src/regexp/onepass.go:500 +0x129
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc003906ff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc003906ff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc0015cca2c)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc003906ff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc003906ff8, 0x1, 0xc003062ab0, 0xc0102376301, 0x280460fee9b8)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc003906ff8, 0x4, 0xc11, 0xc00153a301, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc00153a000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc00078da00, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc002558030, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0039078f8, 0xc002d04a00, 0x13b8, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f41e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 674 [semacquire]:
regexp/syntax.(*compiler).inst(...)
/usr/local/go/src/regexp/syntax/compile.go:173
regexp/syntax.(*compiler).rune(0xc005f9eb30, 0xc00112eb18, 0x1, 0xa, 0xd5, 0xe00000007)
/usr/local/go/src/regexp/syntax/compile.go:267 +0x27c
regexp/syntax.(*compiler).compile(0xc005f9eb30, 0xc005860380, 0x200000001)
/usr/local/go/src/regexp/syntax/compile.go:110 +0x12c
regexp/syntax.(*compiler).compile(0xc005f9eb30, 0xc0058603f0, 0x0)
/usr/local/go/src/regexp/syntax/compile.go:156 +0xb29
regexp/syntax.Compile(0xc0058603f0, 0xc0058603f0, 0x1, 0x1)
/usr/local/go/src/regexp/syntax/compile.go:83 +0x10a
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:178 +0xd7
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc005f9eff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc005f9eff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc00649eedc)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc005f9eff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc005f9eff8, 0x1, 0xc00587c010, 0xc01002f9101, 0x280460fed560)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc005f9eff8, 0x4, 0x231, 0xc003384d01, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc003384000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203001, 0xc00092a270, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc00568aff0, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0016fd8f8, 0xc002ff9000, 0xdca, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f42a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 675 [semacquire]:
strings.genSplit(0xc005b03b00, 0xcd9, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x42fa20, 0x253, 0x253)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00350d3c0, 0xaa0599, 0x2, 0xc00140db00, 0xcd9, 0x1, 0x253, 0xc0063e6b40)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00140db00, 0xcd9, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00350d518, 0xc0032a87e0, 0x56, 0xc0018f4fc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4360, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 676 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc0047e2f40, 0xc0031012c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0031013c0, 0xaa04f9, 0x2, 0xc004e3f500, 0x9a2, 0xc004e3f500, 0xc0031013b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004e3f500, 0x9a2, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003101518, 0xc00229c820, 0x37, 0xc001272780)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4420, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 677 [semacquire]:
strings.genSplit(0xc000d5c000, 0x7dc, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc003a58000, 0x19f, 0x19f)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003beb3c0, 0xaa04fd, 0x2, 0xc002efd000, 0x7dc, 0x1, 0x19f, 0xc0007623c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002efd000, 0x7dc, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003beb518, 0xc00229cf00, 0x56, 0xc0018f5200)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f44e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 679 [semacquire]:
github.com/fatih/set.newTS(...)
/home/develop/go/src/github.com/fatih/set/set_ts.go:15
github.com/fatih/set.New(...)
/home/develop/go/src/github.com/fatih/set/set.go:56
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0028693c0, 0xaa04f9, 0x2, 0xc003f6aa80, 0x9b9, 0xc003f6aa80, 0xc0028693b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:74 +0x52
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003f6aa80, 0x9b9, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002869518, 0xc000762420, 0x3a, 0xc002293f40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4660, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 680 [semacquire]:
golang.org/x/net/html/atom.String(0xc00206b0ea, 0x8, 0xf16, 0xf16, 0xc00206b0f4)
/home/develop/go/src/golang.org/x/net/html/atom/atom.go:77 +0xc6
golang.org/x/net/html.(*Tokenizer).Token(0xc0035ee270, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x12a
golang.org/x/net/html.(*parser).parse(0xc0035ee340, 0xc00209e540, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0007623e0, 0x0, 0x0, 0x0, 0xc002e41200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0007623e0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc002e418f8, 0xc001d4ac00, 0x1181, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4720, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 681 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1b42874f0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002f7f560, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002f7f550, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc004001000, 0x768, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00189b4a0, 0x47, 0xc004001000, 0x768, 0xc004001000, 0x768, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00189b4a0, 0x47, 0xc004001000, 0x768, 0xaa0378, 0x1, 0xc004001000, 0x768, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f47e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 682 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe13400d010, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0033aad00, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0033aacf0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002c15000, 0x71c, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00189b180, 0x45, 0xc002c15000, 0x71c, 0xc002c15000, 0x71c, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00189b180, 0x45, 0xc002c15000, 0x71c, 0xaa0378, 0x1, 0xc002c15000, 0x71c, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f48a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 415 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000897e60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:301 +0x215
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 685 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe208001830, 0x7fe2080010a0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc003299370, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc003299360, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003312000, 0x41f, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001fef577, 0x3c, 0xc003312000, 0x41f, 0xc003312000, 0x41f, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001fef577, 0x3c, 0xc003312000, 0x41f, 0xaa0378, 0x1, 0xc003312000, 0x41f, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4ae0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 686 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2340a0330, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0036ba5c0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0036ba5b0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003270300, 0x63b, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0016a8c80, 0x44, 0xc003270300, 0x63b, 0xc003270300, 0x63b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0016a8c80, 0x44, 0xc003270300, 0x63b, 0xaa0378, 0x1, 0xc003270300, 0x63b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4ba0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 687 [semacquire]:
strings.Replace(0xc002242000, 0xe86, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0xe, 0xc002242000, 0xe86)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0035098f8, 0xc001aec000, 0xe83, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4c60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 688 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc002d2af70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x18a
golang.org/x/net/html.(*parser).parse(0xc002d2b040, 0xc00543ee00, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7b620, 0xc001cfe3c0, 0x0, 0x0, 0x0, 0x0, 0xf01, 0xc001cfe390)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc001cfe390, 0x0, 0x0, 0x0, 0x1000, 0xc002ea9000, 0xfa6, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:93 +0x76
github.com/jaytaylor/html2text.FromString(0xc002ea9000, 0xfa6, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc001d93700)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc004617518, 0xc002f718a0, 0xc0000b2780)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4d20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 689 [semacquire]:
strings.NewReader(...)
/usr/local/go/src/strings/reader.go:154
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0014998f8, 0xc0022e7400, 0x21eb, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:87 +0xf9
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4de0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 690 [semacquire]:
golang.org/x/net/html.NewTokenizerFragment(0xb7c280, 0xc005a64cc0, 0x0, 0x0, 0x20)
/home/develop/go/src/golang.org/x/net/html/token.go:1213 +0x5d
golang.org/x/net/html.NewTokenizer(...)
/home/develop/go/src/golang.org/x/net/html/token.go:1201
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc005a64cc0, 0x0, 0x0, 0x0, 0xc001b4b200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2337 +0x41
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc005a64cc0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc001b4b8f8, 0xc001b77000, 0x168f, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4ea0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 691 [semacquire]:
strings.Replace(0xc00270e000, 0xf51, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0x13, 0xc00270e000, 0xf51)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc000e3f8f8, 0xc000f96000, 0xf4f, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4f60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 693 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe16c001210, 0x7fe16c0013b0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000351750, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc000351720, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0017bea00, 0x6df, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002dde08d, 0x32, 0xc0017bea00, 0x6df, 0xc0017bea00, 0x6df, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002dde08d, 0x32, 0xc0017bea00, 0x6df, 0xaa0378, 0x1, 0xc0017bea00, 0x6df, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f50e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 696 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe21c0014a0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000351130, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc000351120, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001415c00, 0x69b, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000a9e300, 0x3d, 0xc001415c00, 0x69b, 0xc001415c00, 0x69b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000a9e300, 0x3d, 0xc001415c00, 0x69b, 0xaa0378, 0x1, 0xc001415c00, 0x69b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f5320, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 697 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1ac002150, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00124c260, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00124c250, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001ba2000, 0xdc5, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002176198, 0x24, 0xc001ba2000, 0xdc5, 0xc001ba2000, 0xdc5, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002176198, 0x24, 0xc001ba2000, 0xdc5, 0xaa0378, 0x1, 0xc001ba2000, 0xdc5, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f53e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 698 [semacquire]:
strings.genSplit(0xc0033cb000, 0x7a1, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc00521c000, 0x1b1, 0x1b1)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00116b3c0, 0xaa04d9, 0x2, 0xc0050db000, 0x7a1, 0x1, 0x1b1, 0xc00229c880)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0050db000, 0x7a1, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00116b518, 0xc001cfd820, 0x39, 0xc0023e3e00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f54a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 699 [semacquire]:
github.com/gigawattio/window.Rolling(0xc005e55a80, 0x1a1, 0x1a1, 0x3, 0x0, 0x1a0, 0xc005e55a80)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003d71518, 0xc002f70900, 0xc002f712c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f5560, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 700 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe14800e310, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00156caf0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00156cae0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000ad7500, 0xa1d, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001fb630d, 0x49, 0xc000ad7500, 0xa1d, 0xc000ad7500, 0xa1d, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001fb630d, 0x49, 0xc000ad7500, 0xa1d, 0xaa0378, 0x1, 0xc000ad7500, 0xa1d, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f5620, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 702 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe150005170, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002af5610, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002af5600, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001db8900, 0x870, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0023e2540, 0x3a, 0xc001db8900, 0x870, 0xc001db8900, 0x870, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0023e2540, 0x3a, 0xc001db8900, 0x870, 0xaa0378, 0x1, 0xc001db8900, 0x870, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f57a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 705 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe234002aa0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001be5b30, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001be5b20, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001417100, 0x6af, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0016a8870, 0x4e, 0xc001417100, 0x6af, 0xc001417100, 0x6af, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0016a8870, 0x4e, 0xc001417100, 0x6af, 0xaa0378, 0x1, 0xc001417100, 0x6af, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f59e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 706 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/jaytaylor/html2text.FromHTMLNode(0xc00193b340, 0x0, 0x0, 0x0, 0x0, 0xc00193b340, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:81 +0x10d
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc001dc4ed0, 0x0, 0x0, 0x0, 0xa80, 0xc00182ca80, 0x9c8, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc00182ca80, 0x9c8, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc000fff950)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001ce9518, 0xc000c444e0, 0xc000c44de0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f5aa0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 707 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe18c002be0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc004050680, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc004050670, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003440e00, 0x342, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002176c40, 0x3c, 0xc003440e00, 0x342, 0xc003440e00, 0x342, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002176c40, 0x3c, 0xc003440e00, 0x342, 0xaa0378, 0x1, 0xc003440e00, 0x342, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f5b60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 708 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc002cb2800, 0x739, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:81 +0x208
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc002cb2800, 0x739, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc002cb2800, 0x739, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc002cb2800, 0x739, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc002cb2800, 0x739, 0x0, 0x0, 0x739, 0xc002cb2800, 0x739)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc002cb2800, 0x739, 0x739, 0x800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002502af0, 0x66, 0xc002cb2800, 0x739, 0xc002cb2800, 0x739, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002502af0, 0x66, 0xc002cb2800, 0x739, 0xaa0378, 0x1, 0xc002cb2800, 0x739, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f5c20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 709 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1580013c0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0030c6bd0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0030c6bc0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002ea6c00, 0x5b7, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002502c40, 0x62, 0xc002ea6c00, 0x5b7, 0xc002ea6c00, 0x5b7, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002502c40, 0x62, 0xc002ea6c00, 0x5b7, 0xaa0378, 0x1, 0xc002ea6c00, 0x5b7, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f5ce0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 710 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00656d140, 0xc002a612c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002a613c0, 0xaa04f9, 0x2, 0xc000d5d800, 0x712, 0x1, 0x18f, 0xc005ca6cc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000d5d800, 0x712, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002a61518, 0xc00115b980, 0x2d, 0xc000daccf0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f5da0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 711 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc001e52000, 0x13fd, 0xc00149b218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc001e52000, 0x13fd, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00149b3c0, 0xaa0503, 0x2, 0xc001e52000, 0x13fd, 0x4, 0x435, 0xc005d7a5c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001e52000, 0x13fd, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00149b518, 0xc0007630a0, 0x2b, 0xc000114e10)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f5e60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 712 [semacquire]:
regexp.(*Regexp).ReplaceAllString(0xc000149540, 0xc00066e580, 0xc32, 0xaa04bf, 0x2, 0x1, 0x1)
/usr/local/go/src/regexp/regexp.go:560 +0x135
github.com/jaytaylor/html2text.FromHTMLNode(0xc0011d5030, 0x0, 0x0, 0x0, 0x0, 0xc0011d5030, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:80 +0x191
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc00424b9b0, 0x0, 0x0, 0x0, 0x1300, 0xc001d3f900, 0x1080, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc001d3f900, 0x1080, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc001b7e8f0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00149b518, 0xc00115bb20, 0xc0010006c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f5f20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 713 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0015e9600, 0xd35, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:59 +0x21d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0015e9600, 0xd35, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0015e9600, 0xd35, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0015e9600, 0xd35, 0x0, 0x0, 0xd35, 0xc0015e9600, 0xd35)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0015e9600, 0xd35, 0xd35, 0x1300)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0011235a0, 0x17, 0xc0015e9600, 0xd35, 0xc0015e9600, 0xd35, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0011235a0, 0x17, 0xc0015e9600, 0xd35, 0xaa0378, 0x1, 0xc0015e9600, 0xd35, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74000, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 714 [semacquire]:
strings.genSplit(0xc004ba1000, 0x15cf, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc005d53500, 0x48f, 0x48f)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc005fc13c0, 0xaa0599, 0x2, 0xc005074800, 0x15cf, 0x1, 0x48f, 0xc0004e6fa0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005074800, 0x15cf, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc005fc1518, 0xc0004e6c00, 0x15, 0xc003890b40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f740c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 715 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2241c5be0, 0x7fe2241bcff0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0008fda10, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0008fda00, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00285f800, 0x7db, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000b6cc11, 0x1a, 0xc00285f800, 0x7db, 0xc00285f800, 0x7db, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000b6cc11, 0x1a, 0xc00285f800, 0x7db, 0xaa0378, 0x1, 0xc00285f800, 0x7db, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74180, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 716 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003aa3588, 0xaa04f9, 0x2, 0xc00473c900, 0xf7, 0x1, 0x37, 0xc0059f54c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc003aa3518, 0xc00039f0e0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74240, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 717 [runnable]:
strings.Index(0xc00062af80, 0x19de, 0xaa0378, 0x1, 0x4db)
/usr/local/go/src/strings/strings.go:1028 +0x67a
strings.genSplit(0xc00062af80, 0x19de, 0xaa0378, 0x1, 0x0, 0x4db, 0x2434, 0xc000629500, 0x19de)
/usr/local/go/src/strings/strings.go:260 +0xf3
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002c6b588, 0xaa04f9, 0x2, 0xc000629500, 0x19de, 0xc0054a22b0, 0xc002389920, 0xc0054a22b0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc002c6b518, 0xc005946db0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:528 +0xfd
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc002c6b518, 0xc00039fb40, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 718 [semacquire]:
github.com/gigawattio/window.Rolling(0xc005be8000, 0x2d4, 0x2d4, 0x3, 0x0, 0x2d3, 0xc005be8000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003813518, 0xc002226d20, 0xc003383980)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f743c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 719 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1a4227670, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0028a7180, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0028a7170, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00381e000, 0x665, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0031e0d80, 0x38, 0xc00381e000, 0x665, 0xc00381e000, 0x665, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0031e0d80, 0x38, 0xc00381e000, 0x665, 0xaa0378, 0x1, 0xc00381e000, 0x665, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 720 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2241cd440, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0035ec0c0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0035ec0b0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003d2a800, 0x786, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002eeb450, 0x48, 0xc003d2a800, 0x786, 0xc003d2a800, 0x786, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002eeb450, 0x48, 0xc003d2a800, 0x786, 0xaa0378, 0x1, 0xc003d2a800, 0x786, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74540, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 721 [semacquire]:
regexp.(*Regexp).ReplaceAllString(0xc001058be0, 0xc003d22000, 0x130a, 0xaa0378, 0x1, 0x1, 0x30)
/usr/local/go/src/regexp/regexp.go:560 +0x135
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001923518, 0xc001dd20a0, 0xc002227180)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:319 +0x778
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 722 [runnable]:
github.com/PuerkitoBio/goquery.getChildrenWithSiblingType.func1(0xc001899f10, 0xc000176e80)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:617 +0xe5
github.com/PuerkitoBio/goquery.getChildrenWithSiblingType(0xc00294ebd0, 0x0, 0xc0029e8070, 0x0, 0xc00627e408, 0x1, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:659 +0xba
github.com/PuerkitoBio/goquery.getSiblingNodes.func2(0x0, 0xc0029e8070, 0x1, 0xc00627e408, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:599 +0x45
github.com/PuerkitoBio/goquery.mapNodes(0xc000622c00, 0x1, 0x1, 0xc0028e30f0, 0x7fe260fef080, 0xc001f73080, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:693 +0xf6
github.com/PuerkitoBio/goquery.getSiblingNodes(0xc000622c00, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc00627e410, ...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:598 +0x101
github.com/PuerkitoBio/goquery.(*Selection).Siblings(0xc003c4b0e0, 0xc006804630)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:268 +0x59
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc0028e3518, 0xc003c4b0e0, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:517 +0x20f
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0028e3518, 0xc001dd2300, 0x4a)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f746c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 723 [semacquire]:
golang.org/x/net/html.NewTokenizerFragment(0xb7b620, 0xc001098d20, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1215 +0x43
golang.org/x/net/html.NewTokenizer(...)
/home/develop/go/src/golang.org/x/net/html/token.go:1201
golang.org/x/net/html.ParseWithOptions(0xb7b620, 0xc001098d20, 0x0, 0x0, 0x0, 0x0, 0x1401, 0xc0034ada10)
/home/develop/go/src/golang.org/x/net/html/parse.go:2337 +0x41
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0034ada10, 0x0, 0x0, 0x0, 0x1500, 0xc001e60000, 0x14dc, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:93 +0x76
github.com/jaytaylor/html2text.FromString(0xc001e60000, 0x14dc, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc0014f4800)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0021d1518, 0xc0034d1ae0, 0xc00115a820)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74780, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 724 [semacquire]:
strings.genSplit(0xc002c73d40, 0x231, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x231, 0xc002c73b00, 0x231)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00214f400, 0xaa04f9, 0x2, 0xc002c73b00, 0x231, 0xc00214f240, 0x42d308, 0x8)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords.func1(0x5, 0xc00416e450)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:174 +0x86
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc005b1e870, 0xc00214f2c8, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords(0xc00214f388)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:173 +0xb0
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc00214f388, 0xc0029fda10, 0xaa04f9, 0x2, 0x1, 0xffffffffffffffff, 0x400, 0x208, 0x209)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:43 +0xf5
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc00214f518, 0xc0029fda10, 0xaa04f9, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74840, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 725 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc00452e820, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc00452e8f0, 0xc00688c230, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc000fefd00, 0x0, 0x0, 0x0, 0xc00018b200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc000fefd00, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00018b8f8, 0xc001172c00, 0x10ea, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74900, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 726 [semacquire]:
regexp.(*Regexp).replaceAll(0xc0001fb220, 0x0, 0x0, 0x0, 0xc005616000, 0x16a0, 0x2, 0xc003937218, 0xc000512a38, 0x7de1b0, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc0001fb220, 0xc005616000, 0x16a0, 0xaa0378, 0x1, 0xc001884020, 0xc000512c40)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/advancedlogic/GoOse.(*outputFormatter).getOutputText(0xc003937388, 0xc0054d0100, 0xa)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:121 +0x81
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc003937388, 0xc001b5d860, 0xaa04f9, 0x2, 0x42d308, 0x8, 0x9ecc80, 0x175, 0x176)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:45 +0x103
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc003937518, 0xc001b5d860, 0xaa04f9, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f749c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 727 [semacquire]:
bytes.makeSlice(0xbc, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0066fb198, 0xbc, 0x42ca96)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0066fb198, 0xc005f5f800, 0xbc, 0x28, 0x28, 0xc00424a6c0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000568f50)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000568ee0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc00297d680, 0xc00424a6c0, 0xc0066fb210)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc0066fb518, 0xc00297d680, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:527 +0xc5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0066fb518, 0xc0010b1ea0, 0x64)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74a80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 728 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74b40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:314 +0x5a9
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 730 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc003ed01a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc003ed0270, 0xc0040d8cb0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc000c45bc0, 0x0, 0x0, 0x0, 0xc000cc7200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc000c45bc0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc000cc78f8, 0xc00017d000, 0xfd5, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74cc0, 0x59)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 731 [semacquire]:
bytes.makeSlice(0x18c, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc001b63650, 0x84, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc001b63650, 0xc002410ea0, 0x84, 0x6, 0xffffffffffffffff, 0xaa042b)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc001b63650, 0xc002410ea0, 0x84, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc001b63650, 0xc005054e70, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:169 +0x3a3
golang.org/x/net/html.render1(0xb84d20, 0xc001b63650, 0xc005054cb0, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001b63650, 0xc005054c40, 0xb84d20, 0xc001b63650)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc001b63650, 0xc005054c40, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0041f3d70, 0x194002145, 0x1251a20, 0xc0010b0e80, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74d80, 0x59)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 732 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc00045a1a0, 0xc00151ab00, 0xf7)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc00045a1a0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc00045a1a0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc00045a1a0, 0xc001c8c5b0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00608d5a0, 0x0, 0x0, 0x0, 0xc000f7f200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00608d5a0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc000f7f8f8, 0xc00384a000, 0x19f6, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74e40, 0x59)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 733 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc004aae000, 0x15de, 0xc004c03218, 0xc001d7aa80)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc004aae000, 0x15de, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc004c033c0, 0xaa04fd, 0x2, 0xc004aae000, 0x15de, 0x1, 0x41b, 0xc004908ba0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004aae000, 0x15de, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004c03518, 0xc004c38a40, 0x4e, 0xc00456c640)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74f00, 0x59)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 734 [semacquire]:
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc00000b600, 0xf7, 0x2, 0xc005ee5e30, 0x3200000020, 0xc005ee5ed0, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc00000b600, 0xf7, 0xaa0378, 0x1, 0xc005ab7e00, 0x4)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc005ee6100, 0xc0053f20e0, 0xc001aa0000, 0xc005ee5f68)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005ee6100, 0xc0053f2070, 0xc000000001, 0x9)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc005ee70c0, 0xc0053f2070, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:229 +0x376
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc005ee70c0, 0xc0053f2070, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005ee70c0, 0xc0056856c0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc005ee70c0, 0xc0056856c0, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc005ee70c0, 0xc0056856c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc005ee70c0, 0xc0056856c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005ee70c0, 0xc0051a2770, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002a630c0, 0xc0051a2770, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002a630c0, 0xc0051a2770, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005ee70c0, 0xc0051a2690, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002a630c0, 0xc0051a2690, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002a630c0, 0xc0051a2690, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005ee70c0, 0xc004e28310, 0xc000081500, 0x7fe260fee2f0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002a630c0, 0xc004e28310, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002a630c0, 0xc004e28310, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc005ee70c0, 0xc004e28230, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002a630c0, 0xc004e28230, 0x54b170, 0xc002eec750)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc004e28230, 0x0, 0x0, 0x0, 0x0, 0xc004e28230, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0007bb260, 0x0, 0x0, 0x0, 0x1300, 0xc00461d300, 0x114d, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc00461d300, 0x114d, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x6339db1400000000)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002a63518, 0xc002226540, 0xc0032a9720)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74fc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 735 [semacquire]:
bytes.makeSlice(0xe99, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0041f2270, 0x349, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0041f2270, 0xc00568c000, 0x349, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc0041f2270, 0xc00568c000, 0x349, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc0041f2270, 0xc002b90070, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc0041f2270, 0xc00524b490, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc0041f2270, 0xc00524a930, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc0041f2270, 0xc00524a850, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc0041f2270, 0xc00524a150, 0xb84d20, 0xc0041f2270)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc0041f2270, 0xc00524a150, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0041f20c0, 0x1939e63c5, 0x1251a20, 0xc00656c340, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75080, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 736 [semacquire]:
strings.Replace(0xc003b64000, 0x3ce6, 0xaa042f, 0x2, 0xaa0af0, 0x3, 0x11, 0x0, 0x2)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:175 +0x72
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc004c058f8, 0xc003b64000, 0x3ce6, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75140, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 737 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc001e84380, 0xd0b, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x2e6
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc001e84380, 0xd0b, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc001e84380, 0xd0b, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc001e84380, 0xd0b, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc001e84380, 0xd0b, 0x0, 0x0, 0xd0b, 0xc001e84380, 0xd0b)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc001e84380, 0xd0b, 0xd0b, 0xd80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002b7d600, 0x3b, 0xc001e84380, 0xd0b, 0xc001e84380, 0xd0b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002b7d600, 0x3b, 0xc001e84380, 0xd0b, 0xaa0378, 0x1, 0xc001e84380, 0xd0b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75200, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 738 [semacquire]:
strings.Replace(0xc005700000, 0x1eac, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0x14, 0xc005700000, 0x1eac)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0010e78f8, 0xc001408000, 0x1ea6, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f752c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 739 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc001c7f5f0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:1004 +0x4fc
golang.org/x/net/html.(*parser).parseCurrentToken(0xc001c7f5f0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc001c7f5f0, 0xc00524b570, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc002226480, 0x0, 0x0, 0x0, 0xc003c8b200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc002226480, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc003c8b8f8, 0xc001a94000, 0x1431, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75380, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 740 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc003d1e000, 0x1411, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:71 +0x129
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc003d1e000, 0x1411, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc003d1e000, 0x1411, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc003d1e000, 0x1411, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc003d1e000, 0x1411, 0x0, 0x0, 0x1411, 0xc003d1e000, 0x1411)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc003d1e000, 0x1411, 0x1411, 0x1800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002f08140, 0x13, 0xc003d1e000, 0x1411, 0xc003d1e000, 0x1411, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002f08140, 0x13, 0xc003d1e000, 0x1411, 0xaa0378, 0x1, 0xc003d1e000, 0x1411, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75440, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 741 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc001c7f6c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc001c7f790, 0xc00524b650, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc000763c20, 0x0, 0x0, 0x0, 0xc00460d200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc000763c20, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00460d8f8, 0xc0007fe380, 0xcfc, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75500, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 742 [semacquire]:
bytes.makeSlice(0x5a4, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc000e8b530, 0x4, 0x2)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc000e8b530, 0xad01c6, 0x4, 0x2, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.render1(0xb84d20, 0xc000e8b530, 0xc005684620, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:228 +0x7b3
golang.org/x/net/html.render1(0xb84d20, 0xc000e8b530, 0xc005684540, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc000e8b530, 0xc005684310, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc000e8b530, 0xc005684230, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc000e8b530, 0xc003b3a150, 0xb84d20, 0xc000e8b530)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc000e8b530, 0xc003b3a150, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0041f1110, 0x10344a194, 0x1251a20, 0xc0022264c0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f755c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 743 [semacquire]:
golang.org/x/net/html.NewTokenizerFragment(0xb7c280, 0xc00608d560, 0x0, 0x0, 0x20)
/home/develop/go/src/golang.org/x/net/html/token.go:1213 +0x5d
golang.org/x/net/html.NewTokenizer(...)
/home/develop/go/src/golang.org/x/net/html/token.go:1201
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00608d560, 0x0, 0x0, 0x0, 0xc004609200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2337 +0x41
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00608d560, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0046098f8, 0xc0014a0c00, 0x101b, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75680, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 744 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc00358b800, 0x168a, 0x5165d7, 0xc002ec6a80)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc00358b800, 0x168a, 0xc0013e6854, 0x4)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0041b13c0, 0xaa04f9, 0x2, 0xc00358b800, 0x168a, 0xc00358b800, 0xc0041b13b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00358b800, 0x168a, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0041b1518, 0xc00137f5a0, 0x2f, 0xc003416030)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75740, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 745 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc00649bd70, 0x1711475ff, 0x1251a20, 0xc003382140, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75800, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 746 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc0041f2750, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00251f518, 0xc0034d1900, 0x41, 0xc000ffef50)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f758c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 747 [semacquire]:
strings.Replace(0xc003eba000, 0x2cc8, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0x18, 0xc003eba000, 0x2cc8)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0045a58f8, 0xc003091000, 0x2cc6, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75980, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 748 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1c842cae0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001486cc0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001486cb0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0028f6c00, 0x11ed, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000ffe38c, 0x2b, 0xc0028f6c00, 0x11ed, 0xc0028f6c00, 0x11ed, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000ffe38c, 0x2b, 0xc0028f6c00, 0x11ed, 0xaa0378, 0x1, 0xc0028f6c00, 0x11ed, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75a40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 749 [semacquire]:
strings.Replace(0xc004cc4000, 0x1eca, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0x1b, 0xc004cc4000, 0x1eca)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0009178f8, 0xc003bb6000, 0x1ec7, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75b00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 750 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc0035ef790, 0xc001fce240, 0x51)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc0035ef790, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0035ef790)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0035ef790, 0xc003645110, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0034d1920, 0x0, 0x0, 0x0, 0xc003507200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0034d1920, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0035078f8, 0xc001769900, 0x1266, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75bc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 751 [semacquire]:
strings.Replace(0xc005719000, 0xddc, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0x5, 0xc005719000, 0xddc)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc000f278f8, 0xc003bbc000, 0xdd7, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75c80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 752 [semacquire]:
bytes.makeSlice(0x2d8, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0035d1050, 0x8, 0x5)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0035d1050, 0xc0038e8490, 0x8, 0x6, 0xffffffffffffffff, 0xaa042b)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc0035d1050, 0xc0038e8490, 0x8, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc0035d1050, 0xc004bd63f0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:169 +0x3a3
golang.org/x/net/html.render1(0xb84d20, 0xc0035d1050, 0xc0032c3730, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc0035d1050, 0xc0051e4380, 0xb84d20, 0xc0035d1050)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc0035d1050, 0xc0051e4380, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc000e8bbf0, 0x19373785a, 0x1251a20, 0xc00135f140, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75d40, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 753 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc003bd00d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc003bd01a0, 0xc003b96070, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0034d1980, 0x0, 0x0, 0x0, 0xc001c47200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0034d1980, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc001c478f8, 0xc0023ee000, 0x135a, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75e00, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 754 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0044d3800, 0x16c6, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x2e6
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0044d3800, 0x16c6, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0044d3800, 0x16c6, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0044d3800, 0x16c6, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0044d3800, 0x16c6, 0x0, 0x0, 0x16c6, 0xc0044d3800, 0x16c6)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0044d3800, 0x16c6, 0x16c6, 0x1800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc005238030, 0x22, 0xc0044d3800, 0x16c6, 0xc0044d3800, 0x16c6, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc005238030, 0x22, 0xc0044d3800, 0x16c6, 0xaa0378, 0x1, 0xc0044d3800, 0x16c6, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75ec0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 755 [semacquire]:
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc003b72360, 0x10a, 0x2, 0xc0023c8198, 0xc00000000a, 0x2, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc003b72360, 0x10a, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0023c90c0, 0xc0063f3c00, 0xc0023c81b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0023c90c0, 0xc0063f3b90, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0023c90c0, 0xc0063f3b90, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0023c90c0, 0xc0063f3b90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0023c90c0, 0xc0063f3b90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0023c90c0, 0xc00277e2a0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00136b0c0, 0xc00277e2a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00136b0c0, 0xc00277e2a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0023c90c0, 0xc00277e1c0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00136b0c0, 0xc00277e1c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00136b0c0, 0xc00277e1c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0023c90c0, 0xc001215b20, 0xc004303180, 0x7fe260fee9b8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00136b0c0, 0xc001215b20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00136b0c0, 0xc001215b20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0023c90c0, 0xc001215a40, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00136b0c0, 0xc001215a40, 0x54b170, 0xc002eecd00)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc001215a40, 0x0, 0x0, 0x0, 0x0, 0xc001215a40, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc001cfe180, 0x0, 0x0, 0x0, 0x1800, 0xc0013a8800, 0x15d8, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0013a8800, 0x15d8, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc003d61aa0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00136b518, 0xc005d7bbe0, 0xc001884d40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f75f80, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 756 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc004e0b300, 0x105b, 0xc003555218, 0xc000151c00)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc004e0b300, 0x105b, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0035553c0, 0xaa0599, 0x2, 0xc004e0b300, 0x105b, 0x1, 0x2b5, 0xc0063e6600)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004e0b300, 0x105b, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003555518, 0xc005d7a200, 0x5c, 0xc0027164e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94060, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 757 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc0033a9e10, 0xc006288490, 0x7)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc0033a9e10, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0033a9e10)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0033a9e10, 0xc00563d180, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7b620, 0xc001c94e40, 0x0, 0x0, 0x0, 0x0, 0xf01, 0xc001c94e10)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc001c94e10, 0x0, 0x0, 0x0, 0x1000, 0xc003411000, 0xf3f, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:93 +0x76
github.com/jaytaylor/html2text.FromString(0xc003411000, 0xf3f, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc0037b8500)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000e05518, 0xc00135f1e0, 0xc00135fba0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94120, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 758 [runnable]:
github.com/andybalholm/cascadia.(*parser).skipWhitespace(0xc000707068, 0xb878e0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:253 +0x17a
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc000707068, 0x88, 0x88, 0x422c63, 0xc002adef20)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:789 +0xd5
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc000707068, 0xc002adef10, 0x4230ca, 0xc000cdced9, 0xad3865, 0xc000707078)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xaa03ab, 0x1, 0x490000c002adef20, 0x10, 0xc0007070f8, 0x42e6b5, 0xc000707148)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xaa03ab, 0x1, 0xc00039e088, 0xc00038e0f0, 0xc000707148)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc00584c750, 0xaa03ab, 0x1, 0xc0027ef980)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).isHighLinkDensity(0xc000707518, 0xc00584c750, 0x2)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:562 +0x55
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0xa, 0xc00584c750)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:636 +0xa4
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc00584c120, 0xc000707320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc000707518, 0xc00596f140, 0xc0027ee300)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc000707518, 0xc00596f140, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc000707518, 0xc00596f140, 0xc00596f140)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f941e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 759 [semacquire]:
strings.genSplit(0xc00056d900, 0x10a2, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc004ecca00, 0x326, 0x326)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002b273c0, 0xaa059b, 0x2, 0xc0045c8000, 0x10a2, 0x1a, 0x326, 0xc005c54980)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0045c8000, 0x10a2, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002b27518, 0xc00522d360, 0x46, 0xc00189b4f0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f942a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 760 [semacquire]:
bytes.makeSlice(0x382, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00158f1f8, 0x302, 0x7)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00158f1f8, 0xc001990380, 0x302, 0x7, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc002b0d340)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc002b0d180)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc0041f2690, 0x0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0x3, 0xc0041f2690)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:634 +0x4a
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc001b4ce40, 0xc00158f320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc00158f518, 0xc002c70a80, 0xc001cfdda0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc00158f518, 0xc002c70a80, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc00158f518, 0xc002c70a80, 0xc002c70a80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94360, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 761 [runnable]:
github.com/fatih/set.(*Set).Has(0xc00039e080, 0xc002a431c8, 0x1, 0x1, 0xc000480000)
/home/develop/go/src/github.com/fatih/set/set_ts.go:82 +0xd4
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002a43400, 0xaa04f9, 0x2, 0xc003a75ef0, 0xe9, 0xc002a43240, 0x42d308, 0x8)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:81 +0x1e7
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords.func1(0x0, 0xc005d58480)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:174 +0x86
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc005d58450, 0xc002a432c8, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords(0xc002a43388)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:173 +0xb0
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc002a43388, 0xc000dd8270, 0xaa04f9, 0x2, 0x42d308, 0x8, 0x9ecc80, 0xc002a43301, 0xc000901100)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:43 +0xf5
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc002a43518, 0xc000dd8270, 0xaa04f9, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94420, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 763 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1480013e0, 0x7fe148004ba0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00357c730, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00357c720, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000a83800, 0x5b3, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0034922c0, 0x32, 0xc000a83800, 0x5b3, 0xc000a83800, 0x5b3, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0034922c0, 0x32, 0xc000a83800, 0x5b3, 0xaa0378, 0x1, 0xc000a83800, 0x5b3, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f945a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 765 [semacquire]:
strings.genSplit(0xc004897000, 0xe0b, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x57, 0xc0050d8000, 0x2007fe238b8e188)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0023953c0, 0xaa04d9, 0x2, 0xc0050d8000, 0xe0b, 0xc0050d8000, 0xc0023953b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0050d8000, 0xe0b, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002395518, 0xc000fef3c0, 0x3e, 0xc001733940)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94720, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 766 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.ToLower(0xc00437f500, 0x9b8, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:600 +0x222
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0006513c0, 0xaa04df, 0x2, 0xc00437f500, 0x9b8, 0x1, 0x211, 0xc0009c6a20)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00437f500, 0x9b8, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000651518, 0xc004c38c60, 0x49, 0xc003650460)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f947e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 767 [semacquire]:
strings.genSplit(0xc003fe4480, 0x444, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0056fe000, 0xff, 0xff)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc004a8b3c0, 0xaa05c9, 0x2, 0xc0030c2480, 0x444, 0x2, 0xff, 0xc00229d860)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0030c2480, 0x444, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004a8b518, 0xc0050480a0, 0x47, 0xc0026dcfa0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f948a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 768 [semacquire]:
github.com/fatih/set.newTS(...)
/home/develop/go/src/github.com/fatih/set/set_ts.go:15
github.com/fatih/set.New(...)
/home/develop/go/src/github.com/fatih/set/set.go:56
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00298b588, 0xaa04f9, 0x2, 0xc002078000, 0x992, 0x2, 0xb, 0xc002f71820)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:74 +0x52
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00298b518, 0xc0032a9ec0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94960, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 769 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.ToLower(0xc0014f9100, 0x660, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:600 +0x222
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0007833c0, 0xaa059b, 0x2, 0xc0014f9100, 0x660, 0x14, 0x159, 0xc0059f44e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0014f9100, 0x660, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000783518, 0xc004c39400, 0x39, 0xc002b7db00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94a20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 770 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).breakLongLines(0xc0053fb0c0, 0xaa04bf, 0x2, 0xdd, 0x2, 0xc0053fa198)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:462 +0x58c
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0053fb0c0, 0xaa04bf, 0x2, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:426 +0x63
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0053fb0c0, 0xc0053f3880, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:298 +0xb1
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0053fb0c0, 0xc0053f3880, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0053fb0c0, 0xc0053f3880, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0053fb0c0, 0xc0053f2bd0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc001aa90c0, 0xc0053f2bd0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001aa90c0, 0xc0053f2bd0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0053fb0c0, 0xc0053f2af0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc001aa90c0, 0xc0053f2af0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001aa90c0, 0xc0053f2af0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0053fb0c0, 0xc0053f23f0, 0xc001aa8d40, 0x46a81e)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc001aa90c0, 0xc0053f23f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001aa90c0, 0xc0053f23f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0053fb0c0, 0xc0053f2310, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001aa90c0, 0xc0053f2310, 0x54b170, 0xc003bd04e0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc0053f2310, 0x0, 0x0, 0x0, 0x0, 0xc0053f2310, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc002485470, 0x0, 0x0, 0x0, 0xa80, 0xc003544000, 0x926, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc003544000, 0x926, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc000514920)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001aa9518, 0xc0032a95a0, 0xc0032a9e20)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94ae0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 771 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc000bd8b60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x18a
golang.org/x/net/html.(*parser).parse(0xc000bd8c30, 0xc0006b7420, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc004cdaaa0, 0x0, 0x0, 0x0, 0xc003059200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc004cdaaa0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0030598f8, 0xc000994000, 0xf80, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94ba0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 772 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1e8294ef0, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0008fcc50, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0008fcc40, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000545000, 0x3b1, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000b6db90, 0x2e, 0xc000545000, 0x3b1, 0xc000545000, 0x3b1, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000b6db90, 0x2e, 0xc000545000, 0x3b1, 0xaa0378, 0x1, 0xc000545000, 0x3b1, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94c60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 773 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc004acc060, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0043d1518, 0xc0027eee40, 0x45, 0xc000fffc70)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94d20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 774 [semacquire]:
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc002fdd8c0, 0x10e, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:560 +0x135
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0060290c0, 0xc003929c00, 0xc0060281b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060290c0, 0xc003929b90, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0060290c0, 0xc003929b90, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0060290c0, 0xc003929b90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0060290c0, 0xc003929b90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060290c0, 0xc0039292d0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0041410c0, 0xc0039292d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0041410c0, 0xc0039292d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060290c0, 0xc0039291f0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0041410c0, 0xc0039291f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0041410c0, 0xc0039291f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060290c0, 0xc003928af0, 0xc004303180, 0x7fe260fef080)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0041410c0, 0xc003928af0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0041410c0, 0xc003928af0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060290c0, 0xc003928a10, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0041410c0, 0xc003928a10, 0x54b170, 0xc000198410)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc003928a10, 0x0, 0x0, 0x0, 0x0, 0xc003928a10, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc000ccb530, 0x0, 0x0, 0x0, 0x1000, 0xc0051e3000, 0xe36, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0051e3000, 0xe36, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000b6d560)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc004141518, 0xc005964ec0, 0xc005dc5d80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94de0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 775 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1b428fd00, 0x7fe1b40018c0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0049c34e0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0049c34d0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00337cb00, 0x546, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002eea5f0, 0x42, 0xc00337cb00, 0x546, 0xc00337cb00, 0x546, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002eea5f0, 0x42, 0xc00337cb00, 0x546, 0xaa0378, 0x1, 0xc00337cb00, 0x546, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94ea0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 776 [semacquire]:
github.com/PuerkitoBio/goquery.findWithMatcher.func1(0x0, 0xc0033f8e00, 0x0, 0x8, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:547 +0x180
github.com/PuerkitoBio/goquery.mapNodes(0xc0039566d8, 0x1, 0x1, 0xc005327330, 0x1, 0x0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:693 +0xf6
github.com/PuerkitoBio/goquery.findWithMatcher(0xc0039566d8, 0x1, 0x1, 0xb856e0, 0xc0010016e0, 0xc0053273b0, 0x753f20, 0xaa3fc2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:543 +0x81
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0048d27b0, 0xaa408e, 0x4, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x85
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc005327518, 0xc0009c63a0, 0x4f, 0xc003650690)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:126 +0x200
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94f60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 777 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc006306ab0, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004143518, 0xc001000b60, 0x4e, 0xc003651310)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95020, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 778 [semacquire]:
bytes.makeSlice(0xe42, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc003f6c2d0, 0x142, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc003f6c2d0, 0xc001db7b80, 0x142, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc003f6c2d0, 0xc001db7b80, 0x142, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc003f6c2d0, 0xc000448b60, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc003f6c2d0, 0xc000448af0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc003f6c2d0, 0xc0004ddea0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc003f6c2d0, 0xc0004dddc0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc003f6c2d0, 0xc0004dd6c0, 0xb84d20, 0xc003f6c2d0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc003f6c2d0, 0xc0004dd6c0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc00143ff80, 0x1941b65f6, 0x1251a20, 0xc0000b3640, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f950e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 779 [semacquire]:
strings.genSplit(0xc00421ad00, 0x810, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x0, 0xc0006b3810, 0xc0018d5360)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0018d53c0, 0xaa05c9, 0x2, 0xc00421a400, 0x810, 0xc00421a400, 0xc0018d53b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00421a400, 0x810, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0018d5518, 0xc003309d80, 0x3d, 0xc001bfb580)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f951a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 780 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.ToLower(0xc000968a80, 0x337, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:600 +0x222
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001b653c0, 0xaa04d9, 0x2, 0xc000968a80, 0x337, 0x1, 0xb9, 0xc00229c740)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000968a80, 0x337, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001b65518, 0xc00000ece0, 0x3d, 0xc000e85340)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95260, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 781 [runnable]:
bytes.(*Buffer).WriteString(0xc005bb2690, 0xaa03f5, 0x2, 0x3, 0xc000142089, 0x0)
/usr/local/go/src/bytes/buffer.go:180 +0x10f
golang.org/x/net/html.render1(0xb84d20, 0xc005bb2690, 0xc001b5a770, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:180 +0x523
golang.org/x/net/html.render1(0xb84d20, 0xc005bb2690, 0xc001b5a690, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc005bb2690, 0xc00452bce0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc005bb2690, 0xc00452bc00, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc005bb2690, 0xc00452b500, 0xb84d20, 0xc005bb2690)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc005bb2690, 0xc00452b500, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc005bb23f0, 0x18f5da2d6, 0x1251a20, 0xc00642ac00, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95320, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 782 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003bef3c0, 0xaa0503, 0x2, 0xc0008a5800, 0x5ad, 0x2, 0x12a, 0xc005a647c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0008a5800, 0x5ad, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003bef518, 0xc003383ee0, 0x45, 0xc00226a230)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f953e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 783 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005bcf6e0, 0xc00298f220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00298f588, 0xaa04f9, 0x2, 0xc00589b500, 0xd8, 0x1, 0x1b, 0xc004cdbe40)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00298f518, 0xc00103f780, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f954a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 784 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc004c39ac0, 0xc00413f2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00413f3c0, 0xaa0599, 0x2, 0xc00366f900, 0x1205, 0x1, 0x372, 0xc0004e6be0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00366f900, 0x1205, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00413f518, 0xc0032a8fe0, 0x4d, 0xc000607630)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95560, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 785 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc002212d80, 0xcc5, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:59 +0x21d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc002212d80, 0xcc5, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc002212d80, 0xcc5, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc002212d80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc002212d80, 0xcc5, 0x0, 0x0, 0xcc5, 0xd80, 0xc002212d80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc002212d80, 0xcc5, 0xcc5, 0xd80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000e84b00, 0x40, 0xc002212d80, 0xcc5, 0xc002212d80, 0xcc5, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000e84b00, 0x40, 0xc002212d80, 0xcc5, 0xaa0378, 0x1, 0xc002212d80, 0xcc5, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95620, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 786 [semacquire]:
regexp/syntax.Parse(0xaa5b70, 0x5, 0xc001eb00d4, 0x7fe260fee901, 0x47f290, 0xc004b41e30)
/usr/local/go/src/regexp/syntax/parse.go:712 +0x54
regexp.compile(0xaa5b70, 0x5, 0xd4, 0x9c5d76, 0xc001eb1248, 0x509d68)
/usr/local/go/src/regexp/regexp.go:170 +0x5a
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
regexp.MustCompile(0xaa5b70, 0x5, 0xaa0378)
/usr/local/go/src/regexp/regexp.go:309 +0x4b
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0017f9518, 0xc00000e400, 0xc000763d00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:324 +0x7f8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f956e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 787 [semacquire]:
github.com/gigawattio/window.Rolling(0xc00468aa80, 0x64a, 0x64a, 0x3, 0x0, 0x649, 0xc00468aa80)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00458f518, 0xc002f70800, 0xc00522cb00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f957a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 788 [runnable]:
strings.Index(0xc004ee4a80, 0x943, 0xaa0378, 0x1, 0x204)
/usr/local/go/src/strings/strings.go:1028 +0x67a
strings.genSplit(0xc004ee4a80, 0x943, 0xaa0378, 0x1, 0x0, 0x204, 0x4f, 0xc004dd5500, 0x7fe2381762f0)
/usr/local/go/src/strings/strings.go:260 +0xf3
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0031c33c0, 0xaa0503, 0x2, 0xc004dd5500, 0x943, 0xc004dd5500, 0xc0031c33b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004dd5500, 0x943, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0031c3518, 0xc004cdb920, 0x38, 0xc002176840)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95860, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 789 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc004d87a40, 0x50d201, 0xc001e1ce80, 0x78, 0x3f)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0057ed518, 0xc000fef820, 0xc005049860)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:285 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95920, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 790 [semacquire]:
strings.Replace(0xc000aba000, 0x284a, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0x18, 0xc000aba000, 0x284a)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0045938f8, 0xc000952000, 0x2847, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f959e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 792 [semacquire]:
regexp.onePassCopy(0xc006306810, 0xc006306810)
/usr/local/go/src/regexp/onepass.go:222 +0x77
regexp.compileOnePass(0xc006306810, 0xc006306810)
/usr/local/go/src/regexp/onepass.go:497 +0x11b
regexp.compile(0xabc10c, 0x11, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc00211f300, 0x0, 0xabc106, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc00211f300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc00211f300, 0x0, 0xb856e0, 0xc00039e008, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc00211f300, 0xc000224f50, 0x0, 0x8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc00211f300, 0x42ca96, 0xc0063067e0, 0x30, 0x28, 0xa4b3a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xabc101, 0x1d, 0xc00211f370, 0x42d308, 0x30, 0xa4b3a0, 0xb85601)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xabc101, 0x1d, 0xc00103ea00, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc003d6a930, 0xabc101, 0x1d, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc00211f518, 0xc0009c6f80, 0xabc101, 0x1d, 0x3d, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:133 +0x3a7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95b60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 793 [runnable]:
sync.(*RWMutex).RUnlock(0xc00039e008)
/usr/local/go/src/sync/rwmutex.go:62 +0x50
github.com/fatih/set.(*Set).Has(0xc00039e000, 0xc000d6d2c0, 0x1, 0x1, 0xc000480100)
/home/develop/go/src/github.com/fatih/set/set_ts.go:86 +0x10c
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000d6d3c0, 0xaa058b, 0x2, 0xc0016ef500, 0x6dc, 0xc0016ef500, 0xc000d6d3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:81 +0x1e7
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0016ef500, 0x6dc, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000d6d518, 0xc000b323c0, 0x46, 0xc000ea65f0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95c20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 794 [semacquire]:
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs.func1(0x0, 0xc006804c00)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:497 +0x83
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc006804bd0, 0xc001bb52c8, 0x7)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs(0xc001bb56b0, 0xc002301fe0, 0xaac03e, 0x7, 0xc002301fe0)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:495 +0xd8
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc001bb56b0, 0xc002301fe0, 0xc002300b60)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:302 +0x39d
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95ce0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 795 [semacquire]:
github.com/fatih/set.newTS(...)
/home/develop/go/src/github.com/fatih/set/set_ts.go:16
github.com/fatih/set.New(...)
/home/develop/go/src/github.com/fatih/set/set.go:56
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002df3588, 0xaa04f9, 0x2, 0xc0037b6a00, 0x122, 0x1, 0x3, 0xc005bcfba0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:74 +0x61
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc002df3518, 0xc0009c7420, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95da0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 796 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0060390c0, 0xc00116fb00, 0x10c, 0x10c, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:430 +0x233
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0060390c0, 0xc004ea8d90, 0xc0060381b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060390c0, 0xc004ea8d20, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0060390c0, 0xc004ea8d20, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0060390c0, 0xc004ea8d20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0060390c0, 0xc004ea8d20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060390c0, 0xc004ea85b0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0036290c0, 0xc004ea85b0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0036290c0, 0xc004ea85b0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060390c0, 0xc004ea84d0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0036290c0, 0xc004ea84d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0036290c0, 0xc004ea84d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060390c0, 0xc005cee850, 0xc003628db8, 0x443c10)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0036290c0, 0xc005cee850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0036290c0, 0xc005cee850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060390c0, 0xc005cee770, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0036290c0, 0xc005cee770, 0x54b170, 0xc003d5d5f0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc005cee770, 0x0, 0x0, 0x0, 0x0, 0xc005cee770, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc004032f90, 0x0, 0x0, 0x0, 0x1300, 0xc0055e8600, 0x1266, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0055e8600, 0x1266, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc0048ba460)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003629518, 0xc00000f480, 0xc000c44440)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95e60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 797 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe20c227e10, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0003ce690, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0003ce670, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00326e000, 0x6b6, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc003354500, 0x31, 0xc00326e000, 0x6b6, 0xc00326e000, 0x6b6, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc003354500, 0x31, 0xc00326e000, 0x6b6, 0xaa0378, 0x1, 0xc00326e000, 0x6b6, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95f20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 798 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00522cba0, 0xc0017a92c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0017a93c0, 0xaa0569, 0x2, 0xc0004ee600, 0x5c6, 0x34, 0x12e, 0xc000fef000)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0004ee600, 0x5c6, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0017a9518, 0xc0063e60a0, 0x4f, 0xc00189b950)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fae000, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 799 [syscall]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe234000c60, 0x0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00076e4f0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00076e4e0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000ca0960, 0x1d3, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00235c100, 0x36, 0xc000ca0960, 0x1d3, 0xc000ca0960, 0x1d3, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00235c100, 0x36, 0xc000ca0960, 0x1d3, 0xaa0378, 0x1, 0xc000ca0960, 0x1d3, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fae0c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 802 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc001c5ea60, 0xc0010e52c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0010e53c0, 0xaa04fd, 0x2, 0xc002fc8400, 0x865, 0x1, 0x19e, 0xc00000e7e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002fc8400, 0x865, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0010e5518, 0xc0034d10e0, 0x43, 0xc000ffe820)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fae300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 803 [runnable]:
github.com/andybalholm/cascadia.compoundSelector.Match(0xc002408c00, 0x2, 0x2, 0x0, 0x0, 0xc002961f80, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:807 +0xc1
github.com/andybalholm/cascadia.SelectorGroup.Match(0xc0021fd040, 0x1, 0x1, 0xc002961f80, 0xc005af8d00)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:933 +0x68
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc002408c20, 0xc002961f80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:128 +0x34
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc002408c20, 0xc002961ce0, 0x0, 0x0, 0x0, 0xc001d32e38, 0x46e4b7, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:133 +0x9e
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc002408c20, 0xc002961c70, 0x0, 0x0, 0x0, 0x10, 0x9ed160, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:133 +0x9e
github.com/andybalholm/cascadia.Selector.MatchAll(0xc002408c20, 0xc002961c70, 0xc001d32f00, 0x747325, 0xc003be6380)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:124 +0x46
github.com/PuerkitoBio/goquery.findWithMatcher.func1(0x0, 0xc002961b90, 0x0, 0xc003be6380, 0x7fe260fedc28)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:547 +0x8e
github.com/PuerkitoBio/goquery.mapNodes(0xc000901330, 0x1, 0x1, 0xc001d33060, 0x1, 0x0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:693 +0xf6
github.com/PuerkitoBio/goquery.findWithMatcher(0xc000901330, 0x1, 0x1, 0xb856e0, 0xc002408c20, 0xc001d33148, 0x54b3c5, 0xb7b900)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:543 +0x81
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc005af8cc0, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x85
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc001d338f8, 0xc001df1800, 0x15ca, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fae3c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 804 [semacquire]:
bytes.makeSlice(0xe00, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0027a50e8, 0x200, 0x400)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).ReadFrom(0xc0027a50e8, 0xb7b620, 0xc0032c6810, 0x203000, 0x203000, 0x203000)
/usr/local/go/src/bytes/buffer.go:202 +0x48
io/ioutil.readAll(0xb7b620, 0xc0032c6810, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:36 +0xe3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc0032c6810, 0xc0027a51d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0032c6810, 0x0, 0x0, 0x0, 0x1a80, 0xc003f0da80, 0x1a5f, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc003f0da80, 0x1a5f, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000cd5230)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0027a5518, 0xc000b326e0, 0xc003382720)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fae480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 805 [semacquire]:
bytes.makeSlice(0x81f, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0036410c0, 0x1, 0x1)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0036410c0, 0x1189d49, 0x1, 0x1, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0036410c0, 0xc0063d9810, 0x69, 0x69, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:440 +0xf7
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0036410c0, 0xc00076d6c0, 0xc0036401b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0036410c0, 0xc00076d650, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0036410c0, 0xc00076d650, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0036410c0, 0xc00076d650, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0036410c0, 0xc00076d650, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0036410c0, 0xc00076c9a0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0027a30c0, 0xc00076c9a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0027a30c0, 0xc00076c9a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0036410c0, 0xc00076c8c0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0027a30c0, 0xc00076c8c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0027a30c0, 0xc00076c8c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0036410c0, 0xc0018417a0, 0x20300000000000, 0x7fe23a6bafff)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0027a30c0, 0xc0018417a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0027a30c0, 0xc0018417a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0036410c0, 0xc0018416c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0027a30c0, 0xc0018416c0, 0x54b170, 0xc000643d40)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc0018416c0, 0x0, 0x0, 0x0, 0x0, 0xc0018416c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc002c8ec90, 0x0, 0x0, 0x0, 0xd80, 0xc000b8f600, 0xd1f, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc000b8f600, 0xd1f, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc000f62c30)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0027a3518, 0xc001c5ffa0, 0xc000c44600)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fae540, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 806 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0039886c0, 0x50d2a1, 0xc000115200, 0x30, 0xaada21)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0037af518, 0xc00000ea40, 0xc001cfd280)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:285 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fae600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 807 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1c416cba0, 0x7fe2181cc780)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0028f1600, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0028f15f0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000c1c400, 0xad2, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00235dd4b, 0x2b, 0xc000c1c400, 0xad2, 0xc000c1c400, 0xad2, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00235dd4b, 0x2b, 0xc000c1c400, 0xad2, 0xaa0378, 0x1, 0xc000c1c400, 0xad2, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fae6c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 808 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc002d2bc70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc002d2bd40, 0xc0041cd570, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0000b3a80, 0x0, 0x0, 0x0, 0xc0037ab200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0000b3a80, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0037ab8f8, 0xc001d40c00, 0x104d, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fae780, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 809 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.ToLower(0xc001bd0000, 0x773, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:600 +0x222
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003dab3c0, 0xaa05c9, 0x2, 0xc001bd0000, 0x773, 0x2, 0x17f, 0xc000feebe0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001bd0000, 0x773, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003dab518, 0xc001885c20, 0x4e, 0xc0026dd630)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fae840, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 810 [semacquire]:
bytes.makeSlice(0x103f, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0051530c0, 0x1, 0x20)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0051530c0, 0x1189cea, 0x1, 0x1, 0x2, 0x20)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0051530c0, 0xaa04bf, 0x2, 0xc005152308, 0x567148)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:440 +0xf7
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0051530c0, 0xc002b94770, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:292 +0x44
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0051530c0, 0xc002b94770, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0051530c0, 0xc002b94770, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0051530c0, 0xc0018a4b60, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0021f10c0, 0xc0018a4b60, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021f10c0, 0xc0018a4b60, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0051530c0, 0xc0018a4a80, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0021f10c0, 0xc0018a4a80, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021f10c0, 0xc0018a4a80, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0051530c0, 0xc0018a4380, 0xc001e94000, 0x7fe260fee9b8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0021f10c0, 0xc0018a4380, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021f10c0, 0xc0018a4380, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0051530c0, 0xc0018a42a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021f10c0, 0xc0018a42a0, 0x54b170, 0xc002d2a340)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc0018a42a0, 0x0, 0x0, 0x0, 0x0, 0xc0018a42a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc00512f440, 0x0, 0x0, 0x0, 0x1000, 0xc002efb000, 0xfbc, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc002efb000, 0xfbc, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc002176dc0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0021f1518, 0xc001c5f300, 0xc001c5fc40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fae900, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 811 [runnable]:
github.com/advancedlogic/GoOse.Parser.setAttr(0xc005bb2720, 0xab2997, 0xc, 0xacdda4, 0x2)
/home/develop/go/src/github.com/advancedlogic/GoOse/parser.go:87 +0x301
github.com/advancedlogic/GoOse.(*ContentExtractor).updateNodeCount(0xc002d83518, 0xc005bb2720, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:508 +0xb1
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc002d83518, 0xc001884600, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:428 +0x4e1
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fae9c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 812 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc000898340, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x18a
golang.org/x/net/html.(*parser).parse(0xc000898410, 0xc0051bc150, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc001884000, 0x0, 0x0, 0x0, 0xc003bc3200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc001884000, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc003bc38f8, 0xc0023a5500, 0x1349, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faea80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 813 [semacquire]:
regexp.(*Regexp).ReplaceAllString(0xc000149540, 0xc004c49000, 0x16b4, 0xaa04bf, 0x2, 0x1, 0x3)
/usr/local/go/src/regexp/regexp.go:560 +0x135
github.com/jaytaylor/html2text.FromHTMLNode(0xc000ded2d0, 0x0, 0x0, 0x0, 0x0, 0xc000ded2d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:80 +0x191
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc002497230, 0x0, 0x0, 0x0, 0x2000, 0xc0012e4000, 0x1b60, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0012e4000, 0x1b60, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc002503ab0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000e23518, 0xc00103eb60, 0xc00103f4e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faeb40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 814 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc003320500, 0x23b5, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:59 +0x21d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc003320500, 0x23b5, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc003320500, 0x23b5, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc003320500, 0x23b5, 0x0, 0x0, 0x23b5, 0xc003320500, 0x23b5)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc003320500, 0x23b5, 0x23b5, 0x2600)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000808e00, 0x20, 0xc003320500, 0x23b5, 0xc003320500, 0x23b5, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000808e00, 0x20, 0xc003320500, 0x23b5, 0xaa0378, 0x1, 0xc003320500, 0x23b5, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faec00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 815 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc003bcc000, 0x1ed0, 0xc0034bf218, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc003bcc000, 0x1ed0, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0034bf3c0, 0xaa04df, 0x2, 0xc003bcc000, 0x1ed0, 0x1, 0x523, 0xc003113aa0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003bcc000, 0x1ed0, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0034bf518, 0xc001c5f260, 0x2b, 0xc000b6d8f0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faecc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 816 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc000c45720, 0xc0037ad2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0037ad3c0, 0xaa04f9, 0x2, 0xc003962000, 0x1153, 0x4, 0x3c0, 0xc0012463c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003962000, 0x1153, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0037ad518, 0xc00000e740, 0x56, 0xc00068cf60)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faed80, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 817 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.ToLower(0xc00368f900, 0x4b9, 0x7544a2, 0x0)
/usr/local/go/src/strings/strings.go:600 +0x222
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0039f7588, 0xaa04f9, 0x2, 0xc00368f900, 0x4b9, 0x0, 0xc0039f73d8, 0xc0027ef208)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc0039f7518, 0xc002c718c0, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:528 +0xfd
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0039f7518, 0xc00000ef40, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faee40, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 818 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc0010b0ee0, 0xc0032272c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0032273c0, 0xaa04f9, 0x2, 0xc001e0fb00, 0x412, 0x4, 0xfe, 0xc0010b03a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001e0fb00, 0x412, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003227518, 0xc0009c7de0, 0x33, 0xc003355500)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faef00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 819 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0012cc000, 0x92b, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:81 +0x208
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0012cc000, 0x92b, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0012cc000, 0x92b, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0012cc000, 0x92b, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc0012cc000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0012cc000, 0x92b, 0x0, 0x0, 0x92b, 0xc00, 0xc0012cc000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0012cc000, 0x92b, 0x92b, 0xc00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0017332c0, 0x3c, 0xc0012cc000, 0x92b, 0xc0012cc000, 0x92b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0017332c0, 0x3c, 0xc0012cc000, 0x92b, 0xaa0378, 0x1, 0xc0012cc000, 0x92b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faefc0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 820 [semacquire]:
github.com/gigawattio/window.Rolling(0xc006907500, 0x30f, 0x30f, 0x3, 0x0, 0x30e, 0xc006907500)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:10 +0x6c
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00386f518, 0xc005dc5e20, 0xc0059f54e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faf080, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 821 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc0022b11e0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:908 +0x1761
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0022b11e0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0022b11e0, 0xc002600070, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0009c7960, 0x0, 0x0, 0x0, 0xc0035f7200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0009c7960, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0035f78f8, 0xc00329ea00, 0x2136, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faf140, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 822 [semacquire]:
strings.Replace(0xc006412000, 0x1e7a, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0x9, 0xc006412000, 0x1e7a)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0021ed8f8, 0xc002208000, 0x1e71, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faf200, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 823 [runnable]:
strings.Replace(0xc0067b8000, 0x1a63, 0xaa04bd, 0x2, 0xaa03a3, 0x1, 0xffffffffffffffff, 0xb7b900, 0xc00005e060)
/usr/local/go/src/strings/strings.go:928 +0x590
github.com/jaytaylor/html2text.FromHTMLNode(0xc001a88070, 0x0, 0x0, 0x0, 0x0, 0xc001a88070, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:81 +0x158
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc005d2bc80, 0x0, 0x0, 0x0, 0x2500, 0xc000b51900, 0x22a6, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc000b51900, 0x22a6, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc003651180)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003183518, 0xc002300d80, 0xc00642b040)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faf2c0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 824 [semacquire]:
github.com/fatih/set.newTS(...)
/home/develop/go/src/github.com/fatih/set/set_ts.go:16
github.com/fatih/set.New(...)
/home/develop/go/src/github.com/fatih/set/set.go:56
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc004ad1400, 0xaa04f9, 0x2, 0xc00607e000, 0x3f6, 0xc004ad1240, 0x42d308, 0x8)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:74 +0x61
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords.func1(0x4, 0xc000593e60)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:174 +0x86
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc002c9c6c0, 0xc004ad12c8, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords(0xc004ad1388)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:173 +0xb0
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc004ad1388, 0xc00477fe00, 0xaa04f9, 0x2, 0xc004ad1518, 0x8, 0x9ecc80, 0xc004ad1301, 0xc0015c26a8)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:43 +0xf5
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc004ad1518, 0xc00477fe00, 0xaa04f9, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faf380, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 825 [semacquire]:
strings.genSplit(0xc003a85300, 0x12d3, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0060c6000, 0x510, 0x510)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0042613c0, 0xaa0569, 0x2, 0xc0052f6000, 0x12d3, 0x4e, 0x510, 0xc005a65dc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0052f6000, 0x12d3, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004261518, 0xc005a65a80, 0x35, 0xc003493c80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faf440, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 826 [semacquire]:
bytes.makeSlice(0xdbf, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc000e37388, 0xcd, 0x61)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc000e37388, 0xc000071ad0, 0xcd, 0x5165d7, 0xc0041cf421, 0xc000dce060)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00201a700)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00201a690)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc001002540)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc001002460)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000f3c150)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc001098690, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000e37518, 0xc004cdaf80, 0x35, 0xc001732c00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faf500, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 827 [semacquire]:
bytes.makeSlice(0x1581, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc005ab3388, 0x1e7, 0xf2)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc005ab3388, 0xc002b49200, 0x1e7, 0x5165d7, 0xc002331e32, 0xc0048bdeb8)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00504c770)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00504c700)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0020e2930)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0020e2850)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0020e2150)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc001ed35f0, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc005ab3518, 0xc000fee3a0, 0x3d, 0xc002b7d000)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faf5c0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 828 [semacquire]:
strings.genSplit(0xc0033a3000, 0xef1, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc00616f000, 0x2e1, 0x2e1)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003a953c0, 0xaa04f9, 0x2, 0xc002431000, 0xef1, 0x4, 0x2e1, 0xc0047e2420)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002431000, 0xef1, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003a95518, 0xc002409f80, 0x46, 0xc000607540)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faf680, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 829 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc00045b6c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc00045b790, 0xc0050541c0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc002300dc0, 0x0, 0x0, 0x0, 0xc003181200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc002300dc0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0031818f8, 0xc003322a00, 0x2350, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faf740, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 830 [semacquire]:
github.com/gigawattio/window.Rolling(0xc006032000, 0x3c4, 0x3c4, 0x3, 0x0, 0x3c3, 0xc006032000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00018f518, 0xc0007626a0, 0xc00039efa0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faf800, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 831 [semacquire]:
strings.Replace(0xc002438000, 0x1c7c, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0xc, 0xc002438000, 0x1c7c)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0042698f8, 0xc002f7a000, 0x1c77, 0x64656c2073616820, 0x65642061206f7420, 0x6e6920656e696c63)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faf8c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 832 [semacquire]:
strings.Replace(0xc004ca2000, 0x1f8c, 0xaa5837, 0x5, 0xaa8176, 0x6, 0x1, 0xc004ca2000, 0x1f8c)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:177 +0x108
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc003bc78f8, 0xc003e52000, 0x1f79, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faf980, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 833 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc006596ea0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc006596f70, 0xc0056c1b20, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7b620, 0xc003d28450, 0x0, 0x0, 0x0, 0x0, 0x1501, 0xc003d28420)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc003d28420, 0x0, 0x0, 0x0, 0x1800, 0xc001fc2000, 0x15d8, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:93 +0x76
github.com/jaytaylor/html2text.FromString(0xc001fc2000, 0x15d8, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc002edbe50)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003b5b518, 0xc001c5fee0, 0xc002f70880)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fafa40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 834 [semacquire]:
strings.Replace(0xc004d5aa00, 0x13b7, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0x13, 0xc004d5aa00, 0x13b7)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc001a1d8f8, 0xc002179500, 0x13b3, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fafb00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 835 [semacquire]:
github.com/fatih/set.newTS(...)
/home/develop/go/src/github.com/fatih/set/set_ts.go:16
github.com/fatih/set.New(...)
/home/develop/go/src/github.com/fatih/set/set.go:56
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000b0f588, 0xaa04f9, 0x2, 0xc00427e300, 0x51, 0x1, 0x45, 0xc00103fea0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:74 +0x61
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc000b0f518, 0xc003112a40, 0x4f)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fafbc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 836 [semacquire]:
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0024252e0, 0x20300000000000, 0x7fe23a6bafff, 0xc002425218, 0x131)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:724 +0x716
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0024252e0, 0xc00047f200, 0x5, 0xc002425230, 0x46620f)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0024252e0, 0xc0024252f8, 0x43b3b7, 0x1252790, 0xc002425200, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xaa64ee, 0x5, 0x7fe238135608, 0xc002425301, 0x10100000042ab7a, 0x0, 0x1cb1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xaa64ee, 0x5, 0x0, 0x0, 0xaa0301)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc00419e420, 0xaa64ee, 0x5, 0x35ea)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).getTitleUnmodified(0xc002425518, 0xc00137ebe0, 0x1cb1, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:57 +0x47
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:128 +0x282
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fafc80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 837 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc000763420, 0xc0017ab2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0017ab3c0, 0xaa058b, 0x2, 0xc002feea80, 0x930, 0xc002feea80, 0xc0017ab3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002feea80, 0x930, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0017ab518, 0xc000fee8a0, 0x3c, 0xc002e2a740)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fafd40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 838 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.glob..func18(0xc000000061, 0x1b00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:372 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectScript(0xc00575a000, 0x1bac, 0x2)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:271 +0x3a2
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00575a000, 0x1bac, 0x0, 0x0, 0x1bac, 0xc00575a000, 0x1bac)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:58 +0x39
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00575a000, 0x1bac, 0x1bac, 0x2000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001732e40, 0x38, 0xc00575a000, 0x1bac, 0xc00575a000, 0x1bac, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001732e40, 0x38, 0xc00575a000, 0x1bac, 0xaa0378, 0x1, 0xc00575a000, 0x1bac, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fafe00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 839 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc004ed7900, 0x1295, 0x1295, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc004ed7900, 0x1295, 0x7544a2, 0x0)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00205b588, 0xaa04f9, 0x2, 0xc004ed7900, 0x1295, 0x641e2d, 0x0, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00205b518, 0xc000762000, 0x65)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fafec0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 840 [runnable]:
strings.Replace(0xc006716000, 0x3cdb, 0xaa582d, 0x5, 0xaa7dce, 0x6, 0xffffffffffffffff, 0xc006716000, 0x3cdb)
/usr/local/go/src/strings/strings.go:928 +0x590
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:178 +0x153
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc000f298f8, 0xc002ae8000, 0x3cd6, 0xc00177ccb4, 0x5, 0xc00177ccba)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faff80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 841 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc002530002, 0x17b9, 0xc002785248, 0x79bc18)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc002530002, 0x17b9, 0x0, 0x0)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002421518, 0xc002227e80, 0xc002489460)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:297 +0x11e
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fce060, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 842 [runnable]:
unicode.ToLower(0xc000000020, 0x6f)
/usr/local/go/src/unicode/letter.go:261 +0x8d
strings.Map(0xae62d0, 0xc001996cc1, 0xbc, 0xbd, 0xc001d7aa80)
/usr/local/go/src/strings/strings.go:501 +0x2cc
strings.ToLower(0xc001996cc0, 0xbd, 0x7544a2, 0x0)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0009f7588, 0xaa04f9, 0x2, 0xc001996cc0, 0xbd, 0x0, 0xc0009f73d8, 0xc001dd2ea8)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc0009f7518, 0xc002559aa0, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:528 +0xfd
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0009f7518, 0xc001dd2040, 0x4f)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fce120, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 843 [semacquire]:
strings.genSplit(0xc003ec9300, 0x1159, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc002818a00, 0x33b, 0x33b)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003dbb3c0, 0xaa0599, 0x2, 0xc004b89900, 0x1159, 0x1, 0x33b, 0xc005bce620)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004b89900, 0x1159, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003dbb518, 0xc001246b40, 0x46, 0xc004e86000)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fce1e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 844 [semacquire]:
github.com/gigawattio/window.Rolling(0xc005420000, 0x711, 0x711, 0x3, 0x0, 0x710, 0xc005420000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:10 +0x6c
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000e0d518, 0xc001885f00, 0xc0010b0780)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fce2a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 845 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0021686c0, 0xd00e1cdb, 0x1251a20, 0xc001c5ee40, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fce360, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 846 [semacquire]:
bytes.makeSlice(0x1b4f, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc005e69a40, 0x119, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc005e69a40, 0xc0019cad80, 0x119, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc005e69a40, 0xc0019cad80, 0x119, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc005e69a40, 0xc00393db90, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc005e69a40, 0xc00393db20, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc005e69a40, 0xc002930850, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc005e69a40, 0xc002930770, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc005e69a40, 0xc004efab60, 0xb84d20, 0xc005e69a40)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc005e69a40, 0xc004efab60, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0019144e0, 0x50d201, 0xc0011da5a0, 0x80, 0x3f)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0011cb518, 0xc0004e6a00, 0xc000b321e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:285 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fce420, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 847 [semacquire]:
bytes.makeSlice(0x84, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc000c4ee10, 0x4, 0x2)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc000c4ee10, 0xad063c, 0x4, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.render1(0xb84d20, 0xc000c4ee10, 0xc0021875e0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:148 +0x1fd
golang.org/x/net/html.render1(0xb84d20, 0xc000c4ee10, 0xc002187420, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc000c4ee10, 0xc0021873b0, 0xb84d20, 0xc000c4ee10)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc000c4ee10, 0xc0021873b0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc000c4ec60, 0x1937a351e, 0x1251a20, 0xc001397360, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fce4e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 848 [semacquire]:
bytes.makeSlice(0x491, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc001505388, 0xe7, 0x7)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc001505388, 0xc001d2d3b0, 0xe7, 0x5165d7, 0xc0010f287d, 0xc0008ffe40)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000ae4c40)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000ae4bd0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000ae40e0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000ae4000)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000acd8f0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc000c4e960, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001505518, 0xc00135f6a0, 0x68, 0xc00497ab0f)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fce5a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 849 [semacquire]:
strings.Replace(0xc006414000, 0x1e04, 0xaa5837, 0x5, 0xaa8176, 0x6, 0x1, 0xc006414000, 0x1e04)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:177 +0x108
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0045258f8, 0xc003e22000, 0x1dff, 0xc000dfdaaa, 0x7, 0xc000dfdab2)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fce660, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 850 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc000fee5a0, 0xc002e452c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002e453c0, 0xaa04f9, 0x2, 0xc001852600, 0x1108, 0xc001852600, 0xc002e453b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001852600, 0x1108, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002e45518, 0xc00135fe00, 0x6b, 0xc00497ac53)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fce720, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 851 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc000bd84e0, 0xc002764000, 0x344)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc000bd84e0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc000bd84e0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc000bd84e0, 0xc0005d3b90, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00115ae20, 0x0, 0x0, 0x0, 0xc002a01200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00115ae20, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc002a018f8, 0xc0023c2000, 0x59db, 0xc00160b950, 0xa9ece0, 0xc00160b9e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fce7e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 852 [runnable]:
internal/reflectlite.typedmemmove(0xa36900, 0xc00346dde0, 0xc00346de10)
/usr/local/go/src/runtime/mbarrier.go:190 +0x4e
internal/reflectlite.Swapper.func9(0x1b4, 0x1b6)
/usr/local/go/src/internal/reflectlite/swapper.go:71 +0xc1
sort.swapRange_func(0xc001e677d0, 0xc00391a3c0, 0x1b4, 0x1b6, 0x1)
/usr/local/go/src/sort/zfuncversion.go:67 +0x4b
sort.rotate_func(0xc001e677d0, 0xc00391a3c0, 0x1b4, 0x1b6, 0x1b7)
/usr/local/go/src/sort/zfuncversion.go:257 +0x82
sort.symMerge_func(0xc001e677d0, 0xc00391a3c0, 0x1b2, 0x1b6, 0x1b9)
/usr/local/go/src/sort/zfuncversion.go:241 +0x364
sort.symMerge_func(0xc001e677d0, 0xc00391a3c0, 0x1ac, 0x1b5, 0x1b9)
/usr/local/go/src/sort/zfuncversion.go:247 +0x2ee
sort.symMerge_func(0xc001e677d0, 0xc00391a3c0, 0x1a0, 0x1b1, 0x1b9)
/usr/local/go/src/sort/zfuncversion.go:247 +0x2ee
sort.symMerge_func(0xc001e677d0, 0xc00391a3c0, 0x188, 0x1ad, 0x1b9)
/usr/local/go/src/sort/zfuncversion.go:247 +0x2ee
sort.symMerge_func(0xc001e677d0, 0xc00391a3c0, 0x188, 0x1d3, 0x1ea)
/usr/local/go/src/sort/zfuncversion.go:244 +0x31d
sort.symMerge_func(0xc001e677d0, 0xc00391a3c0, 0x188, 0x227, 0x24c)
/usr/local/go/src/sort/zfuncversion.go:244 +0x31d
sort.symMerge_func(0xc001e677d0, 0xc00391a3c0, 0x188, 0x2c6, 0x311)
/usr/local/go/src/sort/zfuncversion.go:244 +0x31d
sort.symMerge_func(0xc001e677d0, 0xc00391a3c0, 0x0, 0x280, 0x311)
/usr/local/go/src/sort/zfuncversion.go:247 +0x2ee
sort.stable_func(0xc001e677d0, 0xc00391a3c0, 0x311)
/usr/local/go/src/sort/zfuncversion.go:180 +0x12f
sort.SliceStable(0x9de7e0, 0xc002409e80, 0xc001e677d0)
/usr/local/go/src/sort/slice.go:27 +0xcb
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc005cf8000, 0x731, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:48 +0x1c8
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc005cf8000, 0x731, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc005cf8000, 0x731, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc005cf8000, 0x731, 0x0, 0x0, 0x731, 0xc005cf8000, 0x731)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc005cf8000, 0x731, 0x731, 0x800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0023e31c0, 0x38, 0xc005cf8000, 0x731, 0xc005cf8000, 0x731, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0023e31c0, 0x38, 0xc005cf8000, 0x731, 0xaa0378, 0x1, 0xc005cf8000, 0x731, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fce8a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 853 [semacquire]:
strings.Replace(0xc0063b8000, 0x1d4e, 0xaa5837, 0x5, 0xaa8176, 0x6, 0x1, 0xc0063b8000, 0x1d4e)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:177 +0x108
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00460f8f8, 0xc003e94000, 0x1d4c, 0xc0012259ea, 0x0, 0xc0012259eb)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fce960, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 854 [semacquire]:
bytes.makeSlice(0x3b7, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc001db3208, 0x13, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc001db3208, 0xc000808680, 0x13, 0x5165d7, 0xc005b921e0, 0xc001142780)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc001799f10)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc001799ea0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc001799dc0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc001145560, 0xc001145560, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc001db3518, 0xc0012463e0, 0x4f)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:423 +0x38b
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcea20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 855 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc003a30c00, 0x12b4, 0xc0041b5218, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc003a30c00, 0x12b4, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0041b53c0, 0xaa04f9, 0x2, 0xc003a30c00, 0x12b4, 0x7, 0x395, 0xc003213760)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003a30c00, 0x12b4, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0041b5518, 0xc003213400, 0x6e, 0xc001d2aa80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fceae0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 856 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe16424e680, 0x7fe16424e8f0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0003ac5b0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0003ac590, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003d4f000, 0xe04, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0026dc9b0, 0x45, 0xc003d4f000, 0xe04, 0xc003d4f000, 0xe04, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0026dc9b0, 0x45, 0xc003d4f000, 0xe04, 0xaa0378, 0x1, 0xc003d4f000, 0xe04, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fceba0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 857 [semacquire]:
bytes.makeSlice(0xe00, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0032fb0e8, 0x200, 0x400)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).ReadFrom(0xc0032fb0e8, 0xb7b620, 0xc0041f30b0, 0x203001, 0x203001, 0x203001)
/usr/local/go/src/bytes/buffer.go:202 +0x48
io/ioutil.readAll(0xb7b620, 0xc0041f30b0, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:36 +0xe3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc0041f30b0, 0xc0032fb1d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0041f30b0, 0x0, 0x0, 0x0, 0x3000, 0xc005646000, 0x2b15, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc005646000, 0x2b15, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc002411710)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0032fb518, 0xc002f703a0, 0xc0022262e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcec60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 858 [semacquire]:
github.com/gigawattio/window.Rolling(0xc004b6a000, 0x445, 0x445, 0x3, 0x0, 0x444, 0xc004b6a000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003d73518, 0xc00000f220, 0xc001dd22a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fced20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 859 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003dad3c0, 0xaa04f9, 0x2, 0xc002412000, 0x14ac, 0x6, 0x3cc, 0xc00135fbc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002412000, 0x14ac, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003dad518, 0xc000762fc0, 0x3f, 0xc000ccdb15)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcede0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 860 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc000c18a80, 0x90e, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc000c18a80, 0x90e, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc000c18a80, 0x90e, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc000c18a80, 0x90e, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc000c18a80, 0x90e, 0x0, 0x0, 0x90e, 0xc000c18a80, 0x90e)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc000c18a80, 0x90e, 0x90e, 0xa80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00305ed20, 0x59, 0xc000c18a80, 0x90e, 0xc000c18a80, 0x90e, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00305ed20, 0x59, 0xc000c18a80, 0x90e, 0xaa0378, 0x1, 0xc000c18a80, 0x90e, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fceea0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 861 [runnable]:
github.com/fatih/set.(*Set).Add(0xc001c5ed20, 0xc0024d72c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0024d73c0, 0xaa04d9, 0x2, 0xc00378e600, 0x1183, 0xc00378e600, 0xc0024d73b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00378e600, 0x1183, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0024d7518, 0xc001cfd880, 0x5d, 0xc002b2aa80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcef60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 862 [runnable]:
github.com/fatih/set.(*Set).Add(0xc000feef60, 0xc0024d52c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0024d53c0, 0xaa04f9, 0x2, 0xc0046cec80, 0x1899, 0xc0046cec80, 0xc0024d53b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0046cec80, 0x1899, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0024d5518, 0xc000763e80, 0x39, 0xc000e85800)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf020, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 863 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc004c98a80, 0x93d, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc004c98a80, 0x93d, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc004c98a80, 0x93d, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc004c98a80, 0x93d, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc004c98a80, 0x93d, 0x0, 0x0, 0x93d, 0xc004c98a80, 0x93d)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc004c98a80, 0x93d, 0x93d, 0xa80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00226ae60, 0x48, 0xc004c98a80, 0x93d, 0xc004c98a80, 0x93d, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00226ae60, 0x48, 0xc004c98a80, 0x93d, 0xaa0378, 0x1, 0xc004c98a80, 0x93d, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf0e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 864 [semacquire]:
strings.genSplit(0xc00190a000, 0xa27, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc005cd5900, 0x246, 0x246)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0009133c0, 0xaa0599, 0x2, 0xc002d3f500, 0xa27, 0x1, 0x246, 0xc003113440)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002d3f500, 0xa27, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000913518, 0xc002408060, 0x34, 0xc000e85980)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf1a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 865 [semacquire]:
regexp/syntax.(*compiler).inst(...)
/usr/local/go/src/regexp/syntax/compile.go:173
regexp/syntax.(*compiler).rune(0xc004bdce38, 0xc001bfb718, 0x1, 0xa, 0xd5, 0xe00000007)
/usr/local/go/src/regexp/syntax/compile.go:267 +0x27c
regexp/syntax.(*compiler).compile(0xc004bdce38, 0xc00655aee0, 0x200000001)
/usr/local/go/src/regexp/syntax/compile.go:110 +0x12c
regexp/syntax.(*compiler).compile(0xc004bdce38, 0xc00655afc0, 0x0)
/usr/local/go/src/regexp/syntax/compile.go:156 +0xb29
regexp/syntax.Compile(0xc00655afc0, 0xc00655afc0, 0x1, 0x1)
/usr/local/go/src/regexp/syntax/compile.go:83 +0x10a
regexp.compile(0xabc10c, 0x11, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:178 +0xd7
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc004bdd300, 0x0, 0xabc106, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc004bdd300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc004bdd300, 0x0, 0xb856e0, 0xc00039e0c8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc004bdd300, 0xc0009c8540, 0x0, 0x8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc004bdd300, 0x42ca96, 0xc00191ac90, 0x30, 0x28, 0xa4b3a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xabc101, 0x1d, 0xc004bdd370, 0x42d308, 0x30, 0xa4b3a0, 0xb85601)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xabc101, 0x1d, 0xc001885340, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc00323d2c0, 0xabc101, 0x1d, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc004bdd518, 0xc000c44fa0, 0xabc101, 0x1d, 0x8e, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:133 +0x3a7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf260, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 866 [runnable]:
github.com/fatih/set.(*Set).Has(0xc00039e180, 0xc00123b2c0, 0x1, 0x1, 0xc000480100)
/home/develop/go/src/github.com/fatih/set/set_ts.go:82 +0xd4
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00123b3c0, 0xaa05c9, 0x2, 0xc0024c6800, 0x1667, 0xc0024c6800, 0xc00123b3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:81 +0x1e7
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0024c6800, 0x1667, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00123b518, 0xc000c45240, 0x53, 0xc001fb6720)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf320, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 867 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00608d760, 0xc002d1b220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002d1b588, 0xaa04f9, 0x2, 0xc003210d20, 0xe9, 0x86, 0x565, 0xc00522d4e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc002d1b518, 0xc00103f140, 0x82)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf3e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 868 [semacquire]:
strings.genSplit(0xc0009ae800, 0x1619, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc004fee000, 0x429, 0x429)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0023f73c0, 0xaa04f9, 0x2, 0xc000647000, 0x1619, 0x5, 0x429, 0xc001001b80)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000647000, 0x1619, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0023f7518, 0xc002489d20, 0x31, 0xc0025ffd40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf4a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 869 [runnable]:
strings.Index(0xc000aba000, 0x28df, 0xaa0378, 0x1, 0x780)
/usr/local/go/src/strings/strings.go:1028 +0x67a
strings.genSplit(0xc000aba000, 0x28df, 0xaa0378, 0x1, 0x0, 0x780, 0xc005d68000, 0x781, 0x781)
/usr/local/go/src/strings/strings.go:260 +0xf3
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00049d3c0, 0xaa04d9, 0x2, 0xc004356a80, 0x28df, 0x5, 0x781, 0xc005834020)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004356a80, 0x28df, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00049d518, 0xc001885160, 0x3b, 0xc00322e29e)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf560, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 870 [runnable]:
github.com/andybalholm/cascadia.SelectorGroup.Match(0xc0021fc5a0, 0x1, 0x1, 0xc00249b180, 0xc00249aa00)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:933 +0x68
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc00642bbc0, 0xc00249b180, 0xc002781d00, 0x6, 0x8, 0xc002781d00, 0x6, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:128 +0x34
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc00642bbc0, 0xc001feb490, 0xc002781d00, 0x6, 0x8, 0xc002781d00, 0x6, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:133 +0x9e
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc00642bbc0, 0xc001feb3b0, 0xc002781d00, 0x6, 0x8, 0xc002781d00, 0x6, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:133 +0x9e
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc00642bbc0, 0xc00045d7a0, 0x0, 0x0, 0x0, 0x10, 0x9ed160, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:133 +0x9e
github.com/andybalholm/cascadia.Selector.MatchAll(0xc00642bbc0, 0xc00045d7a0, 0xc000914f50, 0x747325, 0xc00004d180)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:124 +0x46
github.com/PuerkitoBio/goquery.findWithMatcher.func1(0x0, 0xc00045d6c0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:547 +0x8e
github.com/PuerkitoBio/goquery.mapNodes(0xc000901058, 0x1, 0x1, 0xc0009150b0, 0x1, 0x0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:693 +0xf6
github.com/PuerkitoBio/goquery.findWithMatcher(0xc000901058, 0x1, 0x1, 0xb856e0, 0xc00642bbc0, 0xc005bb35c0, 0xc005bb35c0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:543 +0x81
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc005bb3410, 0xaa408e, 0x4, 0xaa04f9)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x85
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:63 +0xd5
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0009158f8, 0xc0022e4f00, 0x2247, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf620, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 871 [semacquire]:
strings.genSplit(0xc00216c000, 0xa0d, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc003ed4500, 0x206, 0x206)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0012413c0, 0xaa05c9, 0x2, 0xc0039e6000, 0xa0d, 0x6, 0x206, 0xc005bce360)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0039e6000, 0xa0d, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001241518, 0xc00103e960, 0x42, 0xc001127810)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf6e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 872 [semacquire]:
bytes.makeSlice(0x9e3, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00123f388, 0x16d, 0x90)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00123f388, 0xc000d90000, 0x16d, 0x5165d7, 0xc003d77e30, 0xc0049e31d0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0012777a0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc001277730)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000c9a0e0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000c9a000)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00134e7e0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc0056a02a0, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00123f518, 0xc00103ecc0, 0x4b, 0xc001127950)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf7a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 873 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0052c8000, 0x9da, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:81 +0x208
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0052c8000, 0x9da, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0052c8000, 0x9da, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0052c8000, 0x9da, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0052c8000, 0x9da, 0x0, 0x0, 0x9da, 0xc0052c8000, 0x9da)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0052c8000, 0x9da, 0x9da, 0xc00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001127ae0, 0x47, 0xc0052c8000, 0x9da, 0xc0052c8000, 0x9da, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001127ae0, 0x47, 0xc0052c8000, 0x9da, 0xaa0378, 0x1, 0xc0052c8000, 0x9da, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf860, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 874 [runnable]:
regexp.(*Regexp).doExecute(0xc000149400, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0013bdb15, 0x7, 0x7, 0x2, ...)
/usr/local/go/src/regexp/exec.go:521 +0x59b
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc0013bdb15, 0x7, 0x2, 0xc00683a538, 0xc004926c80, 0xc001bec5d8, ...)
/usr/local/go/src/regexp/regexp.go:599 +0x1a5
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc0013bdb15, 0x7, 0xaa0378, 0x1, 0xc005ab7e00, 0x4)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00683b0c0, 0xc00509ae00, 0xc001aa04d0, 0xc001bec670)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00683b0c0, 0xc00509ad90, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc001bed0c0, 0xc00509ad90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001bed0c0, 0xc00509ad90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00683b0c0, 0xc00509acb0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc001bed0c0, 0xc00509acb0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001bed0c0, 0xc00509acb0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00683b0c0, 0xc00509a5b0, 0xc003be6380, 0x7fe260fedc28)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc001bed0c0, 0xc00509a5b0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001bed0c0, 0xc00509a5b0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00683b0c0, 0xc00509a4d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001bed0c0, 0xc00509a4d0, 0x54b170, 0xc002d2aa90)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc00509a4d0, 0x0, 0x0, 0x0, 0x0, 0xc00509a4d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc004e599e0, 0x0, 0x0, 0x0, 0x2500, 0xc001ac4a00, 0x2495, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc001ac4a00, 0x2495, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc004982780)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001bed518, 0xc00103f540, 0xc002489b60)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf920, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 875 [runnable]:
strings.Index(0xc003584cf2, 0x509, 0xaa0378, 0x1, 0x3)
/usr/local/go/src/strings/strings.go:1028 +0x67a
strings.genSplit(0xc003584cf2, 0x509, 0xaa0378, 0x1, 0x0, 0x42a, 0xc00475c800, 0x42b, 0x42b)
/usr/local/go/src/strings/strings.go:260 +0xf3
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00269b3c0, 0xaa0503, 0x2, 0xc002678000, 0x187b, 0x5, 0x42b, 0xc002489f80)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002678000, 0x187b, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00269b518, 0xc0034d00a0, 0x6a, 0xc000f07810)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf9e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 876 [semacquire]:
strings.Replace(0xc00224a000, 0x169a, 0xaa3f06, 0x4, 0xaa1c36, 0x3, 0x1, 0xc00224a000, 0x169a)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0024a9518, 0xc001885e20, 0xc0031122e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:316 +0x67b
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcfaa0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 877 [semacquire]:
bytes.makeSlice(0xfdc, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc003d6f388, 0x18a, 0x2c)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc003d6f388, 0xc0010fdba0, 0x18a, 0x5165d7, 0xc0044a4dfe, 0xc002b60b08)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00193d6c0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00193d650)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00193c620)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00193c540)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc002186e00)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc002f07740, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003d6f518, 0xc0022274c0, 0x4b, 0xc000ea7ae0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcfb60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 878 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc005a48000, 0xf60, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:87 +0x379
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc005a48000, 0xf60, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc005a48000, 0xf60, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc005a48000, 0xf60, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc005a48000, 0xf60, 0x0, 0x0, 0xf60, 0xc005a48000, 0xf60)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc005a48000, 0xf60, 0xf60, 0x1000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00226b090, 0x4c, 0xc005a48000, 0xf60, 0xc005a48000, 0xf60, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00226b090, 0x4c, 0xc005a48000, 0xf60, 0xaa0378, 0x1, 0xc005a48000, 0xf60, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcfc20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 879 [runnable]:
regexp.(*Regexp).doExecute(0xc000149400, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc001bfa4c0, 0x3a, 0x0, 0x2, ...)
/usr/local/go/src/regexp/exec.go:521 +0x59b
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc001bfa4c0, 0x3a, 0x2, 0xc0069f81d0, 0x0, 0xc0069f8270, ...)
/usr/local/go/src/regexp/regexp.go:599 +0x1a5
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc001bfa4c0, 0x3a, 0xaa0378, 0x1, 0xc005ab7f00, 0x6)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0069f83d8, 0xc004eadb90, 0xc004a102c0, 0xc0069f8308)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069f83d8, 0xc004eadb20, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0069f90c0, 0xc004eadb20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:155 +0x7cc
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0069f90c0, 0xc004eadb20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069f90c0, 0xc004eada40, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0032250c0, 0xc004eada40, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0032250c0, 0xc004eada40, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069f90c0, 0xc004ead960, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0032250c0, 0xc004ead960, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0032250c0, 0xc004ead960, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069f90c0, 0xc004ead260, 0xc004303180, 0x7fe260fee9b8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0032250c0, 0xc004ead260, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0032250c0, 0xc004ead260, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069f90c0, 0xc004ead180, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0032250c0, 0xc004ead180, 0x54b170, 0xc002eecb60)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc004ead180, 0x0, 0x0, 0x0, 0x0, 0xc004ead180, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc00465d440, 0x0, 0x0, 0x0, 0x1a80, 0xc0069e5a80, 0x1a33, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0069e5a80, 0x1a33, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc0010a94d0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003225518, 0xc005c55460, 0xc0018846a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcfce0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 880 [runnable]:
unicode.ToLower(0xc000000064, 0x69)
/usr/local/go/src/unicode/letter.go:261 +0x8d
strings.Map(0xae62d0, 0xc0015a6831, 0x1757, 0xc001bc1218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:501 +0x2cc
strings.ToLower(0xc0015a6800, 0x1788, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001bc13c0, 0xaa04d9, 0x2, 0xc0015a6800, 0x1788, 0x1, 0x48d, 0xc001397560)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0015a6800, 0x1788, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001bc1518, 0xc000fee020, 0x52, 0xc00068dda0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcfda0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 881 [runnable]:
strings.genSplit(0xc001f8b300, 0x1241, 0xaa0378, 0x1, 0x0, 0x452, 0xc003ab5800, 0x453, 0x453)
/usr/local/go/src/strings/strings.go:264 +0x133
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001bc33c0, 0xaa04df, 0x2, 0xc0009dac00, 0x1241, 0x1, 0x453, 0xc000762520)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0009dac00, 0x1241, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001bc3518, 0xc001884500, 0x28, 0xc0012bc630)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcfe60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 882 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe14400e090, 0x7fe14400eea0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000a7cad0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc000a7cac0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000c1b800, 0xae6, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00061e1e0, 0x56, 0xc000c1b800, 0xae6, 0xc000c1b800, 0xae6, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00061e1e0, 0x56, 0xc000c1b800, 0xae6, 0xaa0378, 0x1, 0xc000c1b800, 0xae6, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcff20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 883 [semacquire]:
bytes.makeSlice(0x1bb0, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0049d13e0, 0x58, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0049d13e0, 0xc002ece2c0, 0x58, 0x6, 0x58, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc0049d13e0, 0xc002ece2c0, 0xaf, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:201 +0xe5
golang.org/x/net/html.render1(0xb84d20, 0xc0049d13e0, 0xc0015ab9d0, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc0049d13e0, 0xc0015ab960, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc0049d13e0, 0xc0015aa2a0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc0049d13e0, 0xc0015aa1c0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc0049d13e0, 0xc0027e60e0, 0xb84d20, 0xc0049d13e0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc0049d13e0, 0xc0027e60e0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0030a9d70, 0x3, 0xc001c862b0, 0x6, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003901518, 0xc0032a8160, 0xc00608ccc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:285 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fee000, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 884 [semacquire]:
bytes.makeSlice(0xa29, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc003a97388, 0x5d, 0x6)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc003a97388, 0xc00045f9e0, 0x5d, 0x5165d7, 0xc001802478, 0xc00226a370)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0017988c0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc001798850)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0003c7ce0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0003c6e00)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0003c6d20)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0003c6620)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc002558ab0, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003a97518, 0xc000c45b80, 0x32, 0xc00112e300)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fee0c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 886 [semacquire]:
strings.genSplit(0xc0020c8a80, 0x290a, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0041c0000, 0x76c, 0x76c)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0026973c0, 0xaa04fd, 0x2, 0xc003842000, 0x290a, 0x1, 0x76c, 0xc001c5ff40)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003842000, 0x290a, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002697518, 0xc0004e7800, 0x50, 0xc0007641e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fee240, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 887 [semacquire]:
regexp.(*Regexp).ReplaceAllString(0xc004b20dc0, 0xc00538a200, 0x1af6, 0xaa0378, 0x1, 0x1, 0x2c)
/usr/local/go/src/regexp/regexp.go:560 +0x135
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00171f518, 0xc0000b3400, 0xc002488ee0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:319 +0x778
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fee300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 888 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1fc21f9d0, 0x7fe1fc002670)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc003779120, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc003779110, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00056c600, 0x113d, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000c527b0, 0x2f, 0xc00056c600, 0x113d, 0xc00056c600, 0x113d, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000c527b0, 0x2f, 0xc00056c600, 0x113d, 0xaa0378, 0x1, 0xc00056c600, 0x113d, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fee3c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 889 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc002873e10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc002873ee0, 0xc00114dc00, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00115b220, 0x0, 0x0, 0x0, 0xc001ce7200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00115b220, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc001ce78f8, 0xc00196c000, 0x2d90, 0xc001225f78, 0x0, 0xc001225f79)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fee480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 890 [semacquire]:
bytes.makeSlice(0xda9, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc000593dd0, 0x9f, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc000593dd0, 0xc000e7d5e0, 0x9f, 0x6, 0xffffffffffffffff, 0xad0184)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc000593dd0, 0xc000e7d5e0, 0x9f, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc000593dd0, 0xc001274d90, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc000593dd0, 0xc001274bd0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc000593dd0, 0xc0012745b0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc000593dd0, 0xc0012744d0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc000593dd0, 0xc0011eae70, 0xb84d20, 0xc000593dd0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc000593dd0, 0xc0011eae70, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc001069a40, 0x50d201, 0xc00207cb69, 0x1ee, 0x3f)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001a03518, 0xc001885a60, 0xc001cfc820)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:285 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fee540, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 891 [semacquire]:
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc0021c38c0, 0x15c, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:560 +0x135
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002d870c0, 0xc004306460, 0xc002d861b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc002d870c0, 0xc0043063f0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc002d870c0, 0xc0043063f0, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002d870c0, 0xc0043063f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002d870c0, 0xc0043063f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc002d870c0, 0xc003d269a0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002d870c0, 0xc003d269a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002d870c0, 0xc003d269a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc002d870c0, 0xc003d268c0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002d870c0, 0xc003d268c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002d870c0, 0xc003d268c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc002d870c0, 0xc003d261c0, 0xc00004d500, 0x7fe260fece98)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002d870c0, 0xc003d261c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002d870c0, 0xc003d261c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc002d870c0, 0xc003d260e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002d870c0, 0xc003d260e0, 0x54b170, 0xc0064d7e10)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc003d260e0, 0x0, 0x0, 0x0, 0x0, 0xc003d260e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0005adaa0, 0x0, 0x0, 0x0, 0x2000, 0xc005492000, 0x1caf, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc005492000, 0x1caf, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc003416150)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002d87518, 0xc005965840, 0xc0063e6140)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fee600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 892 [semacquire]:
golang.org/x/net/html.NewTokenizerFragment(0xb7b620, 0xc0033cc6f0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1215 +0x43
golang.org/x/net/html.NewTokenizer(...)
/home/develop/go/src/golang.org/x/net/html/token.go:1201
golang.org/x/net/html.ParseWithOptions(0xb7b620, 0xc0033cc6f0, 0x0, 0x0, 0x0, 0x0, 0x1c01, 0xc005c91d10)
/home/develop/go/src/golang.org/x/net/html/parse.go:2337 +0x41
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc005c91d10, 0x0, 0x0, 0x0, 0x2000, 0xc005654000, 0x1c62, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:93 +0x76
github.com/jaytaylor/html2text.FromString(0xc005654000, 0x1c62, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc00226a500)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0034bb518, 0xc003383fc0, 0xc0000b27e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fee6c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 893 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc000f1e800, 0x729, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:71 +0x129
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc000f1e800, 0x729, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc000f1e800, 0x729, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc000f1e800, 0x729, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc000f1e800, 0x729, 0x0, 0x0, 0x729, 0xc000f1e800, 0x729)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc000f1e800, 0x729, 0x729, 0x800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0022f23c0, 0x5f, 0xc000f1e800, 0x729, 0xc000f1e800, 0x729, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0022f23c0, 0x5f, 0xc000f1e800, 0x729, 0xaa0378, 0x1, 0xc000f1e800, 0x729, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fee780, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 894 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe11000ef70, 0x7fe11000f120)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00360a080, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00360a070, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002659800, 0x1744, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0022921d5, 0x22, 0xc002659800, 0x1744, 0xc002659800, 0x1744, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0022921d5, 0x22, 0xc002659800, 0x1744, 0xaa0378, 0x1, 0xc002659800, 0x1744, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fee840, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 896 [semacquire]:
strings.Replace(0xc002598000, 0x1957, 0xaa5837, 0x5, 0xaa8176, 0x6, 0x1, 0xc002598000, 0x1957)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:177 +0x108
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0034b78f8, 0xc002519300, 0x1950, 0xc0010ff017, 0x4, 0xc0010ff01c)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fee9c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 897 [semacquire]:
github.com/gigawattio/window.Rolling(0xc0032ec000, 0x2ba, 0x2ba, 0x3, 0x0, 0x2b9, 0xc0032ec000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003bed518, 0xc005a644c0, 0xc00039eb40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001feea80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 898 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc000643380, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:941 +0x1058
golang.org/x/net/html.(*parser).parseCurrentToken(0xc000643380)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc000643380, 0xc0019ce070, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc000763220, 0x0, 0x0, 0x0, 0xc003bc5200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc000763220, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc003bc58f8, 0xc0032b3000, 0x46d7, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001feeb40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 899 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc002010000, 0x1b95, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:87 +0x3aa
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc002010000, 0x1b95, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc002010000, 0x1b95, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc002010000, 0x1b95, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc002010000, 0x1b95, 0x0, 0x0, 0x1b95, 0xc002010000, 0x1b95)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc002010000, 0x1b95, 0x1b95, 0x2500)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00189b7c0, 0x45, 0xc002010000, 0x1b95, 0xc002010000, 0x1b95, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00189b7c0, 0x45, 0xc002010000, 0x1b95, 0xaa0378, 0x1, 0xc002010000, 0x1b95, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001feec00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 900 [semacquire]:
golang.org/x/net/html/atom.String(0xc0063ec0eb, 0x8, 0xf15, 0xf15, 0xc0063ec0f5)
/home/develop/go/src/golang.org/x/net/html/atom/atom.go:77 +0xc6
golang.org/x/net/html.(*Tokenizer).Token(0xc000098d00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x12a
golang.org/x/net/html.(*parser).parse(0xc000098dd0, 0xc0044b7c70, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0063e6060, 0x0, 0x0, 0x0, 0xc0034c1200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0063e6060, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0034c18f8, 0xc0014ba800, 0x17ab, 0xc000f5e540, 0x3, 0x3)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001feecc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 901 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc001dfe400, 0x8a2, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:40 +0x82
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc001dfe400, 0x8a2, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc001dfe400, 0x8a2, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc001dfe400, 0x8a2, 0x0, 0x0, 0x8a2, 0xc001dfe400, 0x8a2)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc001dfe400, 0x8a2, 0x8a2, 0xa80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000deb570, 0x6d, 0xc001dfe400, 0x8a2, 0xc001dfe400, 0x8a2, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000deb570, 0x6d, 0xc001dfe400, 0x8a2, 0xaa0378, 0x1, 0xc001dfe400, 0x8a2, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001feed80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 902 [semacquire]:
strings.Replace(0xc004074000, 0x1b3b, 0xaa042f, 0x2, 0xaa0af0, 0x3, 0x3, 0x0, 0x0)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:175 +0x72
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0045218f8, 0xc004074000, 0x1b3b, 0xc001432c19, 0x2, 0xc001432c1c)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001feee40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 903 [semacquire]:
strings.Replace(0xc003c33000, 0x2bd9, 0xaa042f, 0x2, 0xaa0af0, 0x3, 0x3, 0xc001ed3ce0, 0x0)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:175 +0x72
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0043d78f8, 0xc003c33000, 0x2bd9, 0xc001433181, 0x4, 0xc001433186)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001feef00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 904 [semacquire]:
github.com/jaytaylor/html2text.FromString(0xc001778000, 0x1b9e, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000dad380)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:102 +0x42
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0045b9518, 0xc00229cf60, 0xc000feed40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001feefc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 905 [runnable]:
github.com/fatih/set.(*Set).Has(0xc00039e200, 0xc0053a32c0, 0x1, 0x1, 0xc000480001)
/home/develop/go/src/github.com/fatih/set/set_ts.go:71 +0x169
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0053a33c0, 0xaa04d9, 0x2, 0xc0044f2000, 0x132e, 0x4, 0x3fc, 0xc002300ba0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:81 +0x1e7
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0044f2000, 0x132e, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0053a3518, 0xc004c39780, 0x3f, 0xc005cbd040)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef080, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 906 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe13800e050, 0x7fe13800e230)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001615fc0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001615fb0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00289d600, 0x897, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00305efc0, 0x33, 0xc00289d600, 0x897, 0xc00289d600, 0x897, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00305efc0, 0x33, 0xc00289d600, 0x897, 0xaa0378, 0x1, 0xc00289d600, 0x897, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef140, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 907 [semacquire]:
bytes.makeSlice(0x344, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc000861128, 0x7e, 0xe3)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc000861128, 0xc002220a00, 0x7e, 0xe3, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00239ee00)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc002056c40)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc0035d07b0, 0xaa03ab, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).isHighLinkDensity(0xc000861518, 0xc0035d07b0, 0x2)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:566 +0x95
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0x7, 0xc0035d07b0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:636 +0xa4
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc001d095f0, 0xc000861320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc000861518, 0xc00038f3e0, 0xc001c5f4e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc000861518, 0xc00038f3e0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc000861518, 0xc00038f3e0, 0xc00038f3e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef200, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 908 [semacquire]:
strings.genSplit(0xc005561200, 0x25fc, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x42fa20, 0x7ce, 0x7ce)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0009193c0, 0xaa0503, 0x2, 0xc003424000, 0x25fc, 0x5, 0x7ce, 0xc005834f60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003424000, 0x25fc, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000919518, 0xc001001c40, 0x64, 0xc00327f3b0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef2c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 909 [runnable]:
strings.Index(0xc003b3ca05, 0x1fd, 0xaa0378, 0x1, 0x4)
/usr/local/go/src/strings/strings.go:1028 +0x67a
strings.genSplit(0xc003b3ca05, 0x1fd, 0xaa0378, 0x1, 0x0, 0x273, 0xc006144800, 0x274, 0x274)
/usr/local/go/src/strings/strings.go:260 +0xf3
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0040053c0, 0xaa0503, 0x2, 0xc004d60580, 0xc02, 0x5, 0x274, 0xc004909f20)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004d60580, 0xc02, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004005518, 0xc004cda360, 0x64, 0xc000f06fc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef380, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 910 [runnable]:
github.com/fatih/set.(*Set).Add(0xc0034d1cc0, 0xc00123d2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00123d3c0, 0xaa04f9, 0x2, 0xc004c3f600, 0xd77, 0xc004c3f600, 0xc00123d3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004c3f600, 0xd77, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00123d518, 0xc00103eb00, 0x35, 0xc000e85a92)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef440, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 911 [semacquire]:
strings.genSplit(0xc00398a000, 0x12ef, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x0, 0xc004715d50, 0xc001beb360)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001beb3c0, 0xaa04d9, 0x2, 0xc001e5d900, 0x12ef, 0xc001e5d900, 0xc001beb3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001e5d900, 0x12ef, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001beb518, 0xc002408480, 0x51, 0xc000907b80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef500, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 912 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc003a88c00, 0x11ad, 0xc001a05218, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc003a88c00, 0x11ad, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001a053c0, 0xaa0599, 0x2, 0xc003a88c00, 0x11ad, 0x1, 0x354, 0xc001000a20)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003a88c00, 0x11ad, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001a05518, 0xc00103ef20, 0x78, 0xc000cd3c80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef5c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 913 [runnable]:
strings.genSplit(0xc0038f4a00, 0x2497, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc00646c000, 0xaef, 0xaef)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003da93c0, 0xaa04d9, 0x2, 0xc0019a6000, 0x2497, 0x1, 0xaef, 0xc001396080)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0019a6000, 0x2497, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003da9518, 0xc00229dd40, 0x3d, 0xc0025feec0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef680, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 914 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc000237520, 0xc00649eec0, 0x8)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc000237520, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc000237520)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc000237520, 0xc003e3a5b0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00229d5e0, 0x0, 0x0, 0x0, 0xc0017a7200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00229d5e0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0017a78f8, 0xc002666f00, 0x20d8, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef740, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 915 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.(*Model).Predict(0xc0000b3320, 0xc000a0ec00, 0x11d3, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:153 +0x132
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLangFT(0xc000a0ec00, 0x11d3, 0xaa04f9, 0x2)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesft.go:14 +0x5f
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0014189b8, 0x46, 0xc000a0ec00, 0x11d3, 0xc000a0ec00, 0x11d3, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:12 +0x6b
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0014189b8, 0x46, 0xc000a0ec00, 0x11d3, 0xaa0378, 0x1, 0xc000a0ec00, 0x11d3, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef800, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 916 [semacquire]:
github.com/fatih/set.newTS(...)
/home/develop/go/src/github.com/fatih/set/set_ts.go:15
github.com/fatih/set.New(...)
/home/develop/go/src/github.com/fatih/set/set.go:56
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0021e7400, 0xaa04f9, 0x2, 0xc0048e8000, 0xc9, 0xc0021e7240, 0x42d308, 0x8)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:74 +0x52
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords.func1(0x0, 0xc00118be30)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:174 +0x86
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc00118be00, 0xc0021e72c8, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords(0xc0021e7388)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:173 +0xb0
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc0021e7388, 0xc0019143f0, 0xaa04f9, 0x2, 0x42d308, 0x8, 0x9ecc80, 0x1, 0xc000901630)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:43 +0xf5
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc0021e7518, 0xc0019143f0, 0xaa04f9, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef8c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 917 [semacquire]:
github.com/gigawattio/window.Rolling(0xc001a60000, 0x712, 0x712, 0x3, 0x0, 0x711, 0xc001a60000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003b59518, 0xc001396800, 0xc001397280)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef980, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 918 [semacquire]:
github.com/fatih/set.newTS(...)
/home/develop/go/src/github.com/fatih/set/set_ts.go:15
github.com/fatih/set.New(...)
/home/develop/go/src/github.com/fatih/set/set.go:56
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0033613c0, 0xaa04f9, 0x2, 0xc001b18000, 0xf3b, 0xc001b18000, 0xc0033613b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:74 +0x52
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001b18000, 0xf3b, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003361518, 0xc000fee320, 0x49, 0xc000ffe0f0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fefa40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 919 [semacquire]:
github.com/gigawattio/window.Rolling(0xc004f44f00, 0x241, 0x241, 0x3, 0x0, 0x240, 0xc004f44f00)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0039d3518, 0xc0032a9020, 0xc000b32ca0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fefb00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 920 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc003288c00, 0x54b, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:48 +0x172
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc003288c00, 0x54b, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc003288c00, 0x54b, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc003288c00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc003288c00, 0x54b, 0x0, 0x0, 0x54b, 0x700, 0xc003288c00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc003288c00, 0x54b, 0x54b, 0x700)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000aab110, 0x67, 0xc003288c00, 0x54b, 0xc003288c00, 0x54b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000aab110, 0x67, 0xc003288c00, 0x54b, 0xaa0378, 0x1, 0xc003288c00, 0x54b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fefbc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 922 [semacquire]:
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc001c597b0, 0xb, 0x2, 0xc003a5de30, 0x200000000a, 0xc003a5ded0, ...)
/usr/local/go/src/regexp/regexp.go:597 +0x57
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc001c597b0, 0xb, 0xaa0378, 0x1, 0x42ca96, 0xc00324e650)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc003a5f0c0, 0xc00698b730, 0xc004a102c0, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc003a5f0c0, 0xc00698b6c0, 0x7fe260fee9b8, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc003a5f0c0, 0xc00698b6c0, 0x83, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:249 +0xca
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc003a5f0c0, 0xc00698b6c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc003a5f0c0, 0xc00698b500, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc003a5f0c0, 0xc00698b500, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc003a5f0c0, 0xc00698b500, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc003a5f0c0, 0xc00698b500, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc003a5f0c0, 0xc0058ab9d0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc001d350c0, 0xc0058ab9d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001d350c0, 0xc0058ab9d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc003a5f0c0, 0xc0058ab8f0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc001d350c0, 0xc0058ab8f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001d350c0, 0xc0058ab8f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc003a5f0c0, 0xc0058ab1f0, 0xc003be6380, 0x7fe260fed560)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc001d350c0, 0xc0058ab1f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001d350c0, 0xc0058ab1f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc003a5f0c0, 0xc0058ab110, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001d350c0, 0xc0058ab110, 0x54b170, 0xc00256ef70)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc0058ab110, 0x0, 0x0, 0x0, 0x0, 0xc0058ab110, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc00135a720, 0x0, 0x0, 0x0, 0x1300, 0xc001312c00, 0x10a0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc001312c00, 0x10a0, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0xc000c9daf8, 0x75bd50)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001d35518, 0xc0032a8a80, 0xc005b744a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fefd40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 923 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.ToLower(0xc00074fef0, 0xef, 0x7544a2, 0x0)
/usr/local/go/src/strings/strings.go:600 +0x222
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002445588, 0xaa04f9, 0x2, 0xc00074fef0, 0xef, 0x8, 0x1e, 0xc005834460)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc002445518, 0xc0032a8100, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fefe00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 925 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe130008000, 0x7fe1300081e0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002edb660, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002edb650, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0036da400, 0x8f0, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001feede0, 0x54, 0xc0036da400, 0x8f0, 0xc0036da400, 0x8f0, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001feede0, 0x54, 0xc0036da400, 0x8f0, 0xaa0378, 0x1, 0xc0036da400, 0x8f0, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001feff80, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 926 [semacquire]:
bytes.NewReader(...)
/usr/local/go/src/bytes/reader.go:160
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc0030600c0, 0xc000c851d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:33 +0xc3
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0030600c0, 0x0, 0x0, 0x0, 0xc80, 0xc00066c000, 0xc26, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc00066c000, 0xc26, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc0038aab60)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000c85518, 0xc002f707c0, 0xc002f71040)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffe060, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 927 [semacquire]:
bytes.makeSlice(0x180c, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00291d710, 0x42, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00291d710, 0xc0016a9540, 0x42, 0x6, 0xffffffffffffffff, 0xad0184)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc00291d710, 0xc0016a9540, 0x42, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc00291d710, 0xc002348ee0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc00291d710, 0xc002348d20, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00291d710, 0xc001d4dce0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00291d710, 0xc001d4dc00, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00291d710, 0xc001d4d500, 0xb84d20, 0xc00291d710)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc00291d710, 0xc001d4d500, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc00291d470, 0x1715e6af6, 0x1251a20, 0xc005a654a0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffe120, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 928 [semacquire]:
strings.genSplit(0xc006880000, 0x1051, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x1300, 0x0, 0xc000b99360)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000b993c0, 0xaa04fd, 0x2, 0xc005dd8c00, 0x1051, 0xc005dd8c00, 0xc000b993b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005dd8c00, 0x1051, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000b99518, 0xc00137e020, 0x32, 0xc001bfb380)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffe1e0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 929 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe16024a770, 0x7fe16024b580)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0029ca230, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0029ca220, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0012f6c00, 0xb83, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000bbacd0, 0x41, 0xc0012f6c00, 0xb83, 0xc0012f6c00, 0xb83, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000bbacd0, 0x41, 0xc0012f6c00, 0xb83, 0xaa0378, 0x1, 0xc0012f6c00, 0xb83, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffe2a0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 930 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe164005940, 0x7fe16400b4f0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0003adae0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0003adad0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001f47900, 0x1022, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0003813e0, 0x51, 0xc001f47900, 0x1022, 0xc001f47900, 0x1022, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0003813e0, 0x51, 0xc001f47900, 0x1022, 0xaa0378, 0x1, 0xc001f47900, 0x1022, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffe360, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 931 [semacquire]:
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0028672e0, 0x432e00, 0xc002867210, 0x9a030e24, 0x9d2)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:724 +0x716
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0028672e0, 0xc000878600, 0x5, 0xc002867230, 0x46620f)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0028672e0, 0xc0028672f8, 0x43b3b7, 0x1252790, 0xc002867200, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xaa64ee, 0x5, 0x7fe23804aa38, 0xc002867301, 0x10100000042ab7a, 0x0, 0x1992)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xaa64ee, 0x5, 0x0, 0x0, 0xaa0301)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0040fa780, 0xaa64ee, 0x5, 0x1eba)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).getTitleUnmodified(0xc002867518, 0xc001000140, 0x1992, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:57 +0x47
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:128 +0x282
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffe420, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 932 [semacquire]:
bytes.makeSlice(0x16f3, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc002427388, 0x205, 0x14e)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc002427388, 0xc001f96000, 0x205, 0x5165d7, 0xc003f43dfb, 0xc0014a3d90)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc004175730)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0041756c0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc004174690)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0041745b0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0042cd570)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc00323d8f0, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002427518, 0xc0010b1180, 0x3b, 0xc002e2a140)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffe4e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 933 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc003382560, 0xc003f212c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003f213c0, 0xaa04f9, 0x2, 0xc00360f900, 0x12d9, 0xc00360f900, 0xc003f213b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00360f900, 0x12d9, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003f21518, 0xc0033821e0, 0x3a, 0xc002177ec0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffe5a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 934 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1400114a0, 0x7fe1400122b0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0016286e0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0016286c0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0026e6000, 0xded, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002eebf40, 0x4b, 0xc0026e6000, 0xded, 0xc0026e6000, 0xded, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002eebf40, 0x4b, 0xc0026e6000, 0xded, 0xaa0378, 0x1, 0xc0026e6000, 0xded, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffe660, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 935 [semacquire]:
strings.makeCutsetFunc(0xaa4e4b, 0x5, 0x0)
/usr/local/go/src/strings/strings.go:831 +0xad
strings.TrimLeft(0xc0025143ac, 0x3, 0xaa4e4b, 0x5, 0x0, 0x1)
/usr/local/go/src/strings/strings.go:855 +0x61
golang.org/x/net/html.beforeHeadIM(0xc00045a680, 0x7fe200000000)
/home/develop/go/src/golang.org/x/net/html/parse.go:583 +0x1f4
golang.org/x/net/html.(*parser).parseCurrentToken(0xc00045a680)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc00045a680, 0xc002714070, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc005bcf760, 0x0, 0x0, 0x0, 0xc001167200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc005bcf760, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0011678f8, 0xc00116e000, 0x2b99, 0xc001e615b1, 0x1, 0xc001e615b3)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffe720, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 936 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc002e63000, 0xe08, 0xc004267218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc002e63000, 0xe08, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0042673c0, 0xaa04fd, 0x2, 0xc002e63000, 0xe08, 0x1, 0x309, 0xc001000f40)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002e63000, 0xe08, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004267518, 0xc0023008c0, 0x3a, 0xc002b7cc40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffe7e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 937 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005d7a780, 0xc0017572c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0017573c0, 0xaa04fd, 0x2, 0xc0042498c0, 0x289, 0x1, 0xa5, 0xc005d7a760)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0042498c0, 0x289, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001757518, 0xc0034d0aa0, 0x34, 0xc002b7dcc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffe8a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 938 [semacquire]:
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc000e70d00, 0x74, 0x2, 0xc0048e2198, 0xc00000000a, 0x2, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc000e70d00, 0x74, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0048e30c0, 0xc0026da700, 0xc0048e21b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0048e30c0, 0xc0026da690, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0048e30c0, 0xc0026da690, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0048e30c0, 0xc0026da690, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0048e30c0, 0xc0026da690, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0048e30c0, 0xc001e76540, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00252d0c0, 0xc001e76540, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00252d0c0, 0xc001e76540, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0048e30c0, 0xc001e76460, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00252d0c0, 0xc001e76460, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00252d0c0, 0xc001e76460, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0048e30c0, 0xc00393de30, 0xc003be6a80, 0x7fe260fee2f0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00252d0c0, 0xc00393de30, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00252d0c0, 0xc00393de30, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0048e30c0, 0xc00393dd50, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00252d0c0, 0xc00393dd50, 0x54b170, 0xc0036fc5b0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc00393dd50, 0x0, 0x0, 0x0, 0x0, 0xc00393dd50, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc004cdd440, 0x0, 0x0, 0x0, 0x1800, 0xc0024c6800, 0x1515, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0024c6800, 0x1515, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc001d2b7a0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00252d518, 0xc001000840, 0xc001001180)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffe960, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 939 [semacquire]:
strings.Replace(0xc004758000, 0x16734, 0xaa5837, 0x5, 0xaa8176, 0x6, 0x9, 0xc004758000, 0x16734)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:177 +0x108
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc001a078f8, 0xc002782000, 0x166d9, 0xc00184a6b5, 0x4, 0xc00184a6ba)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffea20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 940 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/jaytaylor/html2text.FromHTMLNode(0xc00118ea10, 0x0, 0x0, 0x0, 0x0, 0xc00118ea10, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:81 +0x10d
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0042e6b10, 0x0, 0x0, 0x0, 0x2500, 0xc003009400, 0x22d6, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc003009400, 0x22d6, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc0006eed20)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002529518, 0xc00135f2e0, 0xc005a659c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffeae0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 941 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc002488920, 0xc0056d12c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0056d13c0, 0xaa05c9, 0x2, 0xc00206f500, 0x14db, 0x5, 0x3cb, 0xc00135e800)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00206f500, 0x14db, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0056d1518, 0xc0050492e0, 0x4e, 0xc002eeab90)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffeba0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 942 [semacquire]:
regexp.onePassCopy(0xc001b62ea0, 0xc001b62ea0)
/usr/local/go/src/regexp/onepass.go:225 +0x5d
regexp.compileOnePass(0xc001b62ea0, 0xc001b62ea0)
/usr/local/go/src/regexp/onepass.go:497 +0x11b
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0024acff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0024acff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7fe23a585400, 0x20300000000000)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0024acff8, 0x43e200, 0xc0024acee0, 0x5199fd, 0xc001948000)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0024acff8, 0x203000, 0xc0024acfc0, 0x42cb96, 0x100000000000000)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0024acff8, 0x46e6ce, 0xc0010a0400, 0xc003be6380, 0x7fe260fedc28, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc00005e060, 0x8a9, 0x2000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc00452e410, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc001b62e70, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0024ad8f8, 0xc0017e6c00, 0x114e, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffec60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 943 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc00321cc80, 0xc20, 0xc003bd5218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc00321cc80, 0xc20, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003bd53c0, 0xaa04fd, 0x2, 0xc00321cc80, 0xc20, 0x1, 0x247, 0xc00522c100)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00321cc80, 0xc20, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003bd5518, 0xc0027ee2a0, 0x7e, 0xc004f62300)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffed20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 944 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc00592f600, 0x807, 0xc004449218, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc00592f600, 0x807, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0044493c0, 0xaa059b, 0x2, 0xc00592f600, 0x807, 0x11, 0x197, 0xc004c39ec0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00592f600, 0x807, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004449518, 0xc0059f5a60, 0x63, 0xc0046665b0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffede0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 945 [semacquire]:
bytes.makeSlice(0x103f, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00363d0c0, 0x1, 0x1)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00363d0c0, 0x1189d0c, 0x1, 0x1, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc00363d0c0, 0xc002bbf600, 0x1f0, 0x1f0, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:440 +0xf7
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00363d0c0, 0xc00203e070, 0xc00363c1b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00363d0c0, 0xc000fbd0a0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc00363d0c0, 0xc000fbd0a0, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00363d0c0, 0xc000fbd0a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00363d0c0, 0xc000fbd0a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00363d0c0, 0xc000f99ea0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000c030c0, 0xc000f99ea0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000c030c0, 0xc000f99ea0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00363d0c0, 0xc000f99dc0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000c030c0, 0xc000f99dc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000c030c0, 0xc000f99dc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00363d0c0, 0xc000f3c460, 0xc003be6380, 0x7fe260fee9b8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000c030c0, 0xc000f3c460, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000c030c0, 0xc000f3c460, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00363d0c0, 0xc000f3c380, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000c030c0, 0xc000f3c380, 0x54b170, 0xc000199d40)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc000f3c380, 0x0, 0x0, 0x0, 0x0, 0xc000f3c380, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0029fcb40, 0x0, 0x0, 0x0, 0x1000, 0xc002fa6000, 0xdde, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc002fa6000, 0xdde, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc000db0cf0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000c03518, 0xc001884140, 0xc000fef840)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffeea0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 946 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000417588, 0xaa04f9, 0x2, 0xc0040f8f70, 0xc3, 0x7, 0x16, 0xc005a653a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc000417518, 0xc004cda0e0, 0x38)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001ffef60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 947 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0021ef3c0, 0xaa04f9, 0x2, 0xc001f56c00, 0x12d5, 0xc001f56c00, 0xc0021ef3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001f56c00, 0x12d5, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0021ef518, 0xc0022261a0, 0x3a, 0xc00235d100)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fff020, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 948 [semacquire]:
bytes.(*Buffer).grow(0xc00603c3d8, 0x1, 0xc005ffe000)
/usr/local/go/src/bytes/buffer.go:128 +0x25a
bytes.(*Buffer).WriteByte(0xc00603c3d8, 0xaa0320, 0x1, 0xc005ffe000)
/usr/local/go/src/bytes/buffer.go:267 +0x8c
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc00603c3d8, 0xc00427e600, 0x56, 0x56, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:433 +0x2d4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00603c3d8, 0xc005574540, 0xc004e89970, 0xc00603c308)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00603c3d8, 0xc0055744d0, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00603d0c0, 0xc0055744d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:155 +0x7cc
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00603d0c0, 0xc0055744d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00603d0c0, 0xc0055743f0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00425f0c0, 0xc0055743f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00425f0c0, 0xc0055743f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00603d0c0, 0xc005574310, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00425f0c0, 0xc005574310, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00425f0c0, 0xc005574310, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00603d0c0, 0xc000a05b90, 0xc004303180, 0x7fe260fef080)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00425f0c0, 0xc000a05b90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00425f0c0, 0xc000a05b90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00603d0c0, 0xc0042ebf10, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00425f0c0, 0xc0042ebf10, 0x54b170, 0xc000198a90)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc0042ebf10, 0x0, 0x0, 0x0, 0x0, 0xc0042ebf10, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc002c716e0, 0x0, 0x0, 0x0, 0x1800, 0xc0045ec800, 0x1518, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0045ec800, 0x1518, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc001d2b0a0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00425f518, 0xc002409860, 0xc00039f620)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fff0e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 949 [semacquire]:
bytes.makeSlice(0x356d, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc005bb3c80, 0x261, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc005bb3c80, 0xc005bc0280, 0x261, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc005bb3c80, 0xc005bc0280, 0x261, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc005bb3c80, 0xc005bbd3b0, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc005bb3c80, 0xc005bbd340, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc005bb3c80, 0xc005bb1880, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc005bb3c80, 0xc005bb17a0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc005bb3c80, 0xc000bb6150, 0xb84d20, 0xc005bb3c80)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc005bb3c80, 0xc000bb6150, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc005bb3ad0, 0x1337d5d7d, 0x1251a20, 0xc005bcf0c0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fff1a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 950 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc0044e2600, 0x5a2, 0xc002081218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc0044e2600, 0x5a2, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0020813c0, 0xaa04df, 0x2, 0xc0044e2600, 0x5a2, 0x1, 0x133, 0xc00229cd40)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0044e2600, 0x5a2, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002081518, 0xc002300240, 0x31, 0xc001287e4e)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fff260, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 951 [semacquire]:
regexp.(*Regexp).replaceAll(0xc001dc7720, 0x0, 0x0, 0x0, 0xc005581800, 0x16f3, 0x2, 0xc001091218, 0xc005581800, 0x16f3, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc001dc7720, 0xc005581800, 0x16f3, 0xaa0378, 0x1, 0x1, 0x3d)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003881518, 0xc0032a86a0, 0xc0032a98c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:319 +0x778
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fff320, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 953 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc0035ee1a0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:901 +0x820
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0035ee1a0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0035ee1a0, 0xc001d9c070, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0032a80e0, 0x0, 0x0, 0x0, 0xc003ef5200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0032a80e0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc003ef58f8, 0xc0009d2a00, 0x14d4, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fff4a0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 954 [semacquire]:
github.com/PuerkitoBio/goquery.newSingleSelection(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:113
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc000e69710, 0xc004dfb388, 0xc000e69710)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x9b
github.com/advancedlogic/GoOse.OpenGraphResolver(0xc002f70580, 0xc002f70580, 0xc002f70580)
/home/develop/go/src/github.com/advancedlogic/GoOse/images.go:276 +0x106
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:151 +0x646
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fff560, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 955 [runnable]:
strings.genSplit(0xc003dd7438, 0x276f, 0xaa0378, 0x1, 0x0, 0x87a, 0xc0069a6000, 0x87b, 0x87b)
/usr/local/go/src/strings/strings.go:264 +0x133
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0043333c0, 0xaa0599, 0x2, 0xc002b57000, 0x2ba7, 0x1, 0x87b, 0xc00135fca0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002b57000, 0x2ba7, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004333518, 0xc00137f120, 0x41, 0xc00281cb40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fff620, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 956 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc0000984e0, 0x1189d00, 0x1)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc0000984e0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0000984e0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0000984e0, 0xc0027e63f0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc003383e20, 0x0, 0x0, 0x0, 0xc003333200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc003383e20, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0033338f8, 0xc000fd1000, 0x16b3, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fff6e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 957 [semacquire]:
bytes.makeSlice(0x661, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00291c240, 0x7, 0x2)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00291c240, 0xad0877, 0x7, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.render1(0xb84d20, 0xc00291c240, 0xc0015bff80, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:148 +0x1fd
golang.org/x/net/html.render1(0xb84d20, 0xc00291c240, 0xc0015bfd50, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00291c240, 0xc0015bfc70, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00291c240, 0xc00092d0a0, 0xb84d20, 0xc00291c240)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc00291c240, 0xc00092d0a0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc002920150, 0x1711d10e5, 0x1251a20, 0xc0047e2fa0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fff7a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 959 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc0035eea90, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:901 +0x820
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0035eea90)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0035eea90, 0xc0023ae070, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0007631e0, 0x0, 0x0, 0x0, 0xc0022ab200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0007631e0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0022ab8f8, 0xc00328c800, 0x265d, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fff920, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 960 [semacquire]:
regexp/syntax.(*compiler).inst(...)
/usr/local/go/src/regexp/syntax/compile.go:173
regexp/syntax.(*compiler).rune(0xc0031cab30, 0xc003355458, 0x1, 0xa, 0xd5, 0xe00000007)
/usr/local/go/src/regexp/syntax/compile.go:267 +0x27c
regexp/syntax.(*compiler).compile(0xc0031cab30, 0xc004e69b20, 0x200000001)
/usr/local/go/src/regexp/syntax/compile.go:110 +0x12c
regexp/syntax.(*compiler).compile(0xc0031cab30, 0xc004e69b90, 0x0)
/usr/local/go/src/regexp/syntax/compile.go:156 +0xb29
regexp/syntax.Compile(0xc004e69b90, 0xc004e69b90, 0x1, 0x1)
/usr/local/go/src/regexp/syntax/compile.go:83 +0x10a
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:178 +0xd7
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0031caff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0031caff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc0064162dc)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0031caff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0031caff8, 0x1, 0xc0051a14c0, 0xc0101d7aa80, 0x280460fed560)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0031caff8, 0x4, 0xf34, 0xc001ae1001, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc001ae1000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203001, 0xc002f54270, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc005187020, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0031cb8f8, 0xc000675900, 0x106a, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fff9e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 961 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc001396840, 0xc0025232c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0025233c0, 0xaa04d9, 0x2, 0xc002216380, 0xd0a, 0x1, 0x2be, 0xc00522c1a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002216380, 0xd0a, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002523518, 0xc000b33360, 0x2f, 0xc0012bc3f0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fffaa0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 962 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc002d61350, 0xead5ed19, 0x1251a20, 0xc00522ca80, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fffb60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 978 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00522ccc0, 0xc00614f220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00614f588, 0xaa04f9, 0x2, 0xc00248ca00, 0x1345, 0x642137, 0x0, 0xc0009f9a30)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00614f518, 0xc005ca6980, 0xb0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087d320, 0x59)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 979 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc003212580, 0xc0022112c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0022113c0, 0xaa04f9, 0x2, 0xc004319500, 0x9ef, 0x2, 0x1f1, 0xc0033834a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004319500, 0x9ef, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002211518, 0xc0033830c0, 0x50, 0xc0042b2230)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087d3e0, 0x59)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 980 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc005925ec0, 0xc005925ec0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc002e89518, 0xc002300540, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:423 +0x38b
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087d4a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 982 [runnable]:
regexp.makeOnePass.func1(0x2, 0xc0032ff8c0, 0x10, 0x10, 0x42ca96)
/usr/local/go/src/regexp/onepass.go:390 +0x9d8
regexp.makeOnePass.func1(0x1, 0xc0032ff8c0, 0x10, 0x10, 0x4)
/usr/local/go/src/regexp/onepass.go:358 +0x5d7
regexp.makeOnePass(0xc005bb3e00, 0xc005bb3e00)
/usr/local/go/src/regexp/onepass.go:448 +0x398
regexp.compileOnePass(0xc005bb3da0, 0xc005bb3da0)
/usr/local/go/src/regexp/onepass.go:500 +0x129
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc002f48ff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc002f48ff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc0032ff8a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc002f48ff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc002f48ff8, 0x1, 0xc0021fc5d0, 0xc003be6380, 0x280460fedc28)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc002f48ff8, 0x4, 0x9a5, 0xc001fcb601, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc001fcb000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203001, 0xc002ffbd40, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc005bb3d70, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc002f498f8, 0xc001d15800, 0x164f, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087d620, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 983 [runnable]:
golang.org/x/net/html.(*Tokenizer).readByte(0xc000108270, 0xc000108274)
/home/develop/go/src/golang.org/x/net/html/token.go:234 +0x440
golang.org/x/net/html.(*Tokenizer).readTagAttrVal(0xc000108270)
/home/develop/go/src/golang.org/x/net/html/token.go:917 +0xd0
golang.org/x/net/html.(*Tokenizer).readTag(0xc000108270, 0x1)
/home/develop/go/src/golang.org/x/net/html/token.go:831 +0xa6
golang.org/x/net/html.(*Tokenizer).readStartTag(0xc000108270, 0x61)
/home/develop/go/src/golang.org/x/net/html/token.go:779 +0x45
golang.org/x/net/html.(*Tokenizer).Next(0xc000108270, 0x1)
/home/develop/go/src/golang.org/x/net/html/token.go:1021 +0x28c
golang.org/x/net/html.(*parser).parse(0xc000108b60, 0xc001ca59d0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2284 +0xad
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc002227a20, 0x0, 0x0, 0x0, 0xc0020f1200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc002227a20, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0020f18f8, 0xc001cb1900, 0x2104, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087d6e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 984 [semacquire]:
golang.org/x/net/html.NewTokenizerFragment(0xb7c280, 0xc00656da60, 0x0, 0x0, 0x20)
/home/develop/go/src/golang.org/x/net/html/token.go:1215 +0x43
golang.org/x/net/html.NewTokenizer(...)
/home/develop/go/src/golang.org/x/net/html/token.go:1201
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00656da60, 0x0, 0x0, 0x0, 0xc00444b200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2337 +0x41
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00656da60, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00444b8f8, 0xc001c6b300, 0x11a6, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087d7a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 985 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc0003876c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x260
golang.org/x/net/html.(*parser).parse(0xc000387790, 0xc0000fd2d0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0000b2840, 0x0, 0x0, 0x0, 0xc0031ed200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0000b2840, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0031ed8f8, 0xc001fd3800, 0x166a, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087d860, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 986 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc001bdb1e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x260
golang.org/x/net/html.(*parser).parse(0xc001bdb2b0, 0xc0014fd6c0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7b620, 0xc0029a6240, 0x0, 0x0, 0x0, 0x0, 0x1801, 0xc0029a6210)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0029a6210, 0x0, 0x0, 0x0, 0x1980, 0xc002022c80, 0x1892, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:93 +0x76
github.com/jaytaylor/html2text.FromString(0xc002022c80, 0x1892, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc000a92680)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003207518, 0xc005d7b960, 0xc001c5e220)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087d920, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 987 [semacquire]:
strings.Replace(0xc00339e000, 0x1ec1, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0x8, 0xc00339e000, 0x1ec1)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00425d8f8, 0xc002750000, 0x1eb8, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087d9e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 988 [runnable]:
regexp.(*Regexp).backtrack(0xc0001fb400, 0x0, 0x0, 0x0, 0xc003737500, 0x90c, 0x1ed, 0x2, 0xc0038fe320, 0x0, ...)
/usr/local/go/src/regexp/backtrack.go:357 +0x318
regexp.(*Regexp).doExecute(0xc0001fb400, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc003737500, 0x90c, 0x0, 0x2, ...)
/usr/local/go/src/regexp/exec.go:535 +0x453
regexp.(*Regexp).replaceAll(0xc0001fb400, 0x0, 0x0, 0x0, 0xc003737500, 0x90c, 0x2, 0xc0023fba58, 0xaa6444, 0x5, ...)
/usr/local/go/src/regexp/regexp.go:599 +0x1a5
regexp.(*Regexp).ReplaceAllString(0xc0001fb400, 0xc003737500, 0x90c, 0xaa0378, 0x1, 0xc002f6ca80, 0x90c)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
_/home/ec2-user/environment/cluster2/cluster2/base.Sanitize(0xaa6444, 0x5, 0xc003737500, 0x90c, 0xaa0378, 0x1, 0xc003737500, 0x90c)
/home/ec2-user/environment/cluster2/cluster2/base/sanitizer.go:44 +0xc9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087daa0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:305 +0x37c
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 989 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Join(0xc005d591d0, 0x3, 0x3, 0xaa0378, 0x1, 0x0, 0xc000479f80)
/usr/local/go/src/strings/strings.go:438 +0x4c8
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00362b518, 0xc001247b20, 0xc0034d04a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:334 +0xa85
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087db60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 990 [semacquire]:
regexp.makeOnePass(0xc000ccb410, 0xc000ccb410)
/usr/local/go/src/regexp/onepass.go:309 +0x20e
regexp.compileOnePass(0xc0011aa030, 0xc0011aa030)
/usr/local/go/src/regexp/onepass.go:500 +0x129
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc00386cff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc00386cff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc003516408)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc00386cff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc00386cff8, 0x1, 0xc002860010, 0xc0039fea80, 0x280460fee9b8)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc00386cff8, 0x4, 0x9e0, 0xc0009b7601, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc0009b7000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203001, 0xc0036088f0, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0041f3fb0, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00386d8f8, 0xc0010d4800, 0x1601, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087dc20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 991 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe150003f30, 0x7fe1500040d0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000ae8de0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc000ae8dc0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0014f8300, 0x6fc, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00335729a, 0x65, 0xc0014f8300, 0x6fc, 0xc0014f8300, 0x6fc, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00335729a, 0x65, 0xc0014f8300, 0x6fc, 0xaa0378, 0x1, 0xc0014f8300, 0x6fc, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087dce0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 398 [semacquire]:
strings.Replace(0xc0042a9300, 0x1868, 0xaa5837, 0x5, 0xaa8176, 0x6, 0x3, 0xc0042a9300, 0x1868)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:177 +0x108
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0019238f8, 0xc0040d4600, 0x1866, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000381680, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 993 [semacquire]:
regexp.(*Regexp).replaceAll(0xc0001fb400, 0x0, 0x0, 0x0, 0xc005cba480, 0x199b, 0x2, 0xc0044e1a58, 0xaa6444, 0x5, ...)
/usr/local/go/src/regexp/regexp.go:642 +0x7ae
regexp.(*Regexp).ReplaceAllString(0xc0001fb400, 0xc005cba480, 0x199b, 0xaa0378, 0x1, 0xc001543500, 0x199b)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
_/home/ec2-user/environment/cluster2/cluster2/base.Sanitize(0xaa6444, 0x5, 0xc005cba480, 0x199b, 0xaa0378, 0x1, 0xc005cba480, 0x199b)
/home/ec2-user/environment/cluster2/cluster2/base/sanitizer.go:44 +0xc9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087de60, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:305 +0x37c
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 994 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00039fe60, 0xc0043cb2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0043cb3c0, 0xaa058b, 0x2, 0xc000b1c000, 0xe5c, 0x5, 0x2ef, 0xc001cfc080)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000b1c000, 0xe5c, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0043cb518, 0xc0009c7cc0, 0x51, 0xc000312420)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087df20, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 995 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.(*Model).Predict(0xc001884e60, 0xc001988e00, 0x35b, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:153 +0x132
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001988e00, 0x35b, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:31 +0xa0
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0026dc000, 0x4c, 0xc001988e00, 0x35b, 0xc001988e00, 0x35b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0026dc000, 0x4c, 0xc001988e00, 0x35b, 0xaa0378, 0x1, 0xc001988e00, 0x35b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716000, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 996 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc00147d000, 0x15ab, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc00147d000, 0x15ab, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc00147d000, 0x15ab, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc00147d000, 0x15ab, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc00147d000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00147d000, 0x15ab, 0x0, 0x0, 0x15ab, 0x1800, 0xc00147d000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00147d000, 0x15ab, 0x15ab, 0x1800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002177b00, 0x3f, 0xc00147d000, 0x15ab, 0xc00147d000, 0x15ab, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002177b00, 0x3f, 0xc00147d000, 0x15ab, 0xaa0378, 0x1, 0xc00147d000, 0x15ab, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0027160c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 997 [semacquire]:
strings.genSplit(0xc004ef8ea0, 0x8f, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x8f, 0xc004ef8e10, 0x8f)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0005db588, 0xaa04f9, 0x2, 0xc004ef8e10, 0x8f, 0x9, 0xc0005db290, 0x42d308)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0x8, 0xc003dae780)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:635 +0x82
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc001d62c60, 0xc0005db320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc0005db518, 0xc0032189f0, 0xc002226f00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc0005db518, 0xc0032189f0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc0005db518, 0xc0032189f0, 0xc0032189f0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716180, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 998 [semacquire]:
github.com/advancedlogic/GoOse.Parser.setAttr(0xc000c4fbc0, 0xab29a3, 0xc, 0xc006289554, 0x3)
/home/develop/go/src/github.com/advancedlogic/GoOse/parser.go:87 +0x301
github.com/advancedlogic/GoOse.(*ContentExtractor).updateScore(0xc001641518, 0xc000c4fbc0, 0xf)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:493 +0xb1
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc001641518, 0xc000c440e0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:427 +0x4ba
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716240, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 999 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe158001360, 0x7fe158005880)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0012539c0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0012539b0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002fc7200, 0x84a, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00235ccc0, 0x3c, 0xc002fc7200, 0x84a, 0xc002fc7200, 0x84a, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00235ccc0, 0x3c, 0xc002fc7200, 0x84a, 0xaa0378, 0x1, 0xc002fc7200, 0x84a, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1000 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc002e6d500, 0xa57, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:71 +0x129
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc002e6d500, 0xa57, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc002e6d500, 0xa57, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc002e6d500, 0xa57, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc002e6d500)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc002e6d500, 0xa57, 0x0, 0x0, 0xa57, 0xc00, 0xc002e6d500)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc002e6d500, 0xa57, 0xa57, 0xc00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002177d80, 0x40, 0xc002e6d500, 0xa57, 0xc002e6d500, 0xa57, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002177d80, 0x40, 0xc002e6d500, 0xa57, 0xaa0378, 0x1, 0xc002e6d500, 0xa57, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0027163c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1001 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe188004040, 0x7fe188004240)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0011d7c00, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0011d7be0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002353800, 0x72d, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001bfb340, 0x33, 0xc002353800, 0x72d, 0xc002353800, 0x72d, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001bfb340, 0x33, 0xc002353800, 0x72d, 0xaa0378, 0x1, 0xc002353800, 0x72d, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1002 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1981c5d40, 0x7fe190005dd0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001d06970, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001d06960, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002486000, 0x78b, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0016a94a0, 0x49, 0xc002486000, 0x78b, 0xc002486000, 0x78b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0016a94a0, 0x49, 0xc002486000, 0x78b, 0xaa0378, 0x1, 0xc002486000, 0x78b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716540, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1003 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1600018b0, 0x7fe160005e70)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc000dff760, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc000dff750, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002b01300, 0x10ae, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0004276f7, 0x41, 0xc002b01300, 0x10ae, 0xc002b01300, 0x10ae, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0004276f7, 0x41, 0xc002b01300, 0x10ae, 0xaa0378, 0x1, 0xc002b01300, 0x10ae, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1004 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe164001890, 0x7fe164005b90)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0011f87b0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0011f87a0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00327d000, 0x785, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0016a9590, 0x41, 0xc00327d000, 0x785, 0xc00327d000, 0x785, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0016a9590, 0x41, 0xc00327d000, 0x785, 0xaa0378, 0x1, 0xc00327d000, 0x785, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0027166c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1005 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1b80019d0, 0x7fe1b80050c0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0008afdb0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0008afda0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003ec7680, 0x476, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc003354240, 0x36, 0xc003ec7680, 0x476, 0xc003ec7680, 0x476, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc003354240, 0x36, 0xc003ec7680, 0x476, 0xaa0378, 0x1, 0xc003ec7680, 0x476, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716780, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1006 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe170002260, 0x7fe170005710)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0021ea570, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0021ea550, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000a23500, 0x6e9, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001049887, 0x65, 0xc000a23500, 0x6e9, 0xc000a23500, 0x6e9, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001049887, 0x65, 0xc000a23500, 0x6e9, 0xaa0378, 0x1, 0xc000a23500, 0x6e9, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716840, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1007 [semacquire]:
regexp/syntax.(*parser).collapse(0xc002c0c840, 0xc005a65fe0, 0x3, 0x4, 0xc003853e12, 0xc003853e30)
/usr/local/go/src/regexp/syntax/parse.go:380 +0x17a
regexp/syntax.(*parser).concat(0xc002c0c840, 0xa)
/usr/local/go/src/regexp/syntax/parse.go:313 +0x166
regexp/syntax.Parse(0xabc10c, 0x11, 0xd4, 0x0, 0x0, 0xabc106)
/usr/local/go/src/regexp/syntax/parse.go:891 +0x1026
regexp.compile(0xabc10c, 0x11, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:170 +0x5a
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc001ced300, 0x0, 0xabc106, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc001ced300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc001ced300, 0x0, 0xb856e0, 0xc00039e1c8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc001ced300, 0xc000cd0c40, 0x0, 0x8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc001ced300, 0x42ca96, 0xc001098cf0, 0x30, 0x28, 0xa4b3a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xabc101, 0x1d, 0xc001ced370, 0x42d308, 0x30, 0xa4b3a0, 0xb85601)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xabc101, 0x1d, 0xc005a65fa0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc002b6d980, 0xabc101, 0x1d, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc001ced518, 0xc003112ca0, 0xabc101, 0x1d, 0x28, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:133 +0x3a7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716900, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1008 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1bc20bde0, 0x7fe1bc20bf30)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0042f2ad0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0042f2ac0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000389080, 0x542, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0025fe340, 0x3a, 0xc000389080, 0x542, 0xc000389080, 0x542, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0025fe340, 0x3a, 0xc000389080, 0x542, 0xaa0378, 0x1, 0xc000389080, 0x542, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0027169c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1009 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002099588, 0xaa04f9, 0x2, 0xc005470000, 0x949, 0xe, 0x2f, 0xc00039e580)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc002099518, 0xc00137f320, 0x64)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716a80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1010 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001a213c0, 0xaa0599, 0x2, 0xc005bf7100, 0x636, 0xc005bf7100, 0xc001a213b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005bf7100, 0x636, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001a21518, 0xc00229d920, 0x3c, 0xc004750f40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716b40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1011 [semacquire]:
bytes.makeSlice(0x600, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc001d370e8, 0x200, 0x200)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).ReadFrom(0xc001d370e8, 0xb7b620, 0xc000e69350, 0x203000, 0x203000, 0x203000)
/usr/local/go/src/bytes/buffer.go:202 +0x48
io/ioutil.readAll(0xb7b620, 0xc000e69350, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:36 +0xe3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc000e69350, 0xc001d371d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc000e69350, 0x0, 0x0, 0x0, 0x1300, 0xc0066a8600, 0x1085, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc0066a8600, 0x1085, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc00600bc40)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001d37518, 0xc005c55620, 0xc0063e6a00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716c00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1012 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe19000e270, 0x7fe19000e290)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00124c850, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00124c840, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0024f7680, 0x443, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000cd4420, 0x2e, 0xc0024f7680, 0x443, 0xc0024f7680, 0x443, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000cd4420, 0x2e, 0xc0024f7680, 0x443, 0xaa0378, 0x1, 0xc0024f7680, 0x443, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716cc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1013 [semacquire]:
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc002eebc20, 0x44, 0xaa0378, 0x1, 0xc005ab7f80, 0x7)
/usr/local/go/src/regexp/regexp.go:560 +0x135
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0048c3dd0, 0xc0056c0770, 0xc001cee370, 0xc0048c3c38)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0048c3dd0, 0xc0056c0700, 0xc000000001, 0xb)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0048c50c0, 0xc0056c0700, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:229 +0x376
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0048c50c0, 0xc0056c0700, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0048c50c0, 0xc0056c0620, 0xc000000001, 0x9)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0048c50c0, 0xc0056c0620, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0048c50c0, 0xc0056c0620, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0048c50c0, 0xc001489180, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0048c50c0, 0xc001489180, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0048c50c0, 0xc001489180, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0048c50c0, 0xc0014884d0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0048c50c0, 0xc0014884d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0048c50c0, 0xc0014884d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0048c50c0, 0xc0014883f0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0048c50c0, 0xc0014883f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0048c50c0, 0xc0014883f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0048c50c0, 0xc001b2e930, 0xc001d7aa80, 0x7fe260fef080)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0048c50c0, 0xc001b2e930, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0048c50c0, 0xc001b2e930, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0048c50c0, 0xc001b2e850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0048c50c0, 0xc001b2e850, 0x54b170, 0xc000386ea0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc001b2e850, 0x0, 0x0, 0x0, 0x0, 0xc001b2e850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc003c4a150, 0x0, 0x0, 0x0, 0x1000, 0xc00580a000, 0xebe, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc00580a000, 0xebe, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc00087b880)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0048c5518, 0xc0000b31c0, 0xc00000ff40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716d80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1014 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe15400df30, 0x7fe15400ed40)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002419000, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002418ff0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0041d2f00, 0x4a2, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00229205e, 0x1c, 0xc0041d2f00, 0x4a2, 0xc0041d2f00, 0x4a2, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00229205e, 0x1c, 0xc0041d2f00, 0x4a2, 0xaa0378, 0x1, 0xc0041d2f00, 0x4a2, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716e40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1015 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc002d01470, 0x203000, 0x203000, 0x440d95, 0xc000622e28)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs.func1(0x0, 0xc002d01470)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:496 +0x67
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc002d01440, 0xc0005b52c8, 0x7)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs(0xc0005b56b0, 0xc00000eb60, 0xaac03e, 0x7, 0xc00000eb60)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:495 +0xd8
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc0005b56b0, 0xc00000eb60, 0xc00039fec0)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:302 +0x39d
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716f00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1016 [semacquire]:
strings.genSplit(0xc003f42a80, 0x93f, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0056dc000, 0x1f3, 0x1f3)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003dbf3c0, 0xaa05c9, 0x2, 0xc004c83500, 0x93f, 0x4, 0x1f3, 0xc00229d760)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004c83500, 0x93f, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003dbf518, 0xc001cfdde0, 0x49, 0xc0016a8370)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002716fc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1017 [semacquire]:
regexp/syntax.(*parser).maybeConcat(0xc00544d2c0, 0xffffffff, 0xc0021f2a58)
/usr/local/go/src/regexp/syntax/parse.go:165 +0x239
regexp/syntax.(*parser).push(0xc00544d2c0, 0xc0040d9960, 0xc0040d9960)
/usr/local/go/src/regexp/syntax/parse.go:136 +0x218
regexp/syntax.(*parser).literal(0xc00544d2c0, 0x74)
/usr/local/go/src/regexp/syntax/parse.go:210 +0x53
regexp/syntax.Parse(0xac0e09, 0x13, 0xd4, 0x0, 0x0, 0xac0dfd)
/usr/local/go/src/regexp/syntax/parse.go:729 +0x2ae
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:170 +0x5a
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0021f2ff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0021f2ff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc00453fff0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0021f2ff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0021f2ff8, 0x1, 0xc002068070, 0xc0102c4a700, 0x280460fee2f0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0021f2ff8, 0x4, 0x65b, 0xc0049f3901, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc0049f3000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc0000985b0, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0028305a0, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0021f38f8, 0xc0027f5500, 0x9a4, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717080, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1018 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe18000ab60, 0x7fe18000bdf0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002fdef40, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002fdef30, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002f0ca80, 0x90c, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000aa96c0, 0x39, 0xc002f0ca80, 0x90c, 0xc002f0ca80, 0x90c, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000aa96c0, 0x39, 0xc002f0ca80, 0x90c, 0xaa0378, 0x1, 0xc002f0ca80, 0x90c, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717140, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1019 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc00447c000, 0xf10, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc00447c000, 0xf10, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc00447c000, 0xf10, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc00447c000, 0xf10, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00447c000, 0xf10, 0x0, 0x0, 0xf10, 0xc00447c000, 0xf10)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00447c000, 0xf10, 0xf10, 0x1300)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0016a87d0, 0x44, 0xc00447c000, 0xf10, 0xc00447c000, 0xf10, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0016a87d0, 0x44, 0xc00447c000, 0xf10, 0xaa0378, 0x1, 0xc00447c000, 0xf10, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717200, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1020 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1bc2621c0, 0x7fe1bc2622f0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001afe6d0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001afe6c0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0003cd600, 0x8ea, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001418cd0, 0x4c, 0xc0003cd600, 0x8ea, 0xc0003cd600, 0x8ea, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001418cd0, 0x4c, 0xc0003cd600, 0x8ea, 0xaa0378, 0x1, 0xc0003cd600, 0x8ea, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0027172c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1021 [semacquire]:
strings.genSplit(0xc002306000, 0x1dbd, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc003ceaa00, 0x525, 0x525)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0037cf3c0, 0xaa04f9, 0x2, 0xc003912000, 0x1dbd, 0x4, 0x525, 0xc0024891e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003912000, 0x1dbd, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0037cf518, 0xc00137e7c0, 0x61, 0xc001049a6b)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717380, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1022 [semacquire]:
regexp/syntax.(*compiler).inst(...)
/usr/local/go/src/regexp/syntax/compile.go:173
regexp/syntax.(*compiler).rune(0xc001032e38, 0xc005c22858, 0x1, 0xa, 0xd5, 0xe00000007)
/usr/local/go/src/regexp/syntax/compile.go:267 +0x27c
regexp/syntax.(*compiler).compile(0xc001032e38, 0xc0040d8b60, 0x200000001)
/usr/local/go/src/regexp/syntax/compile.go:110 +0x12c
regexp/syntax.(*compiler).compile(0xc001032e38, 0xc0040d8c40, 0x0)
/usr/local/go/src/regexp/syntax/compile.go:156 +0xb29
regexp/syntax.Compile(0xc0040d8c40, 0xc0040d8c40, 0x1, 0x1)
/usr/local/go/src/regexp/syntax/compile.go:83 +0x10a
regexp.compile(0xabc10c, 0x11, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:178 +0xd7
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc001033300, 0x0, 0xabc106, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc001033300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc001033300, 0x0, 0xb856e0, 0xc00039e008, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc001033300, 0xc0037c18f0, 0x0, 0x8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc001033300, 0x42ca96, 0xc002f068a0, 0x30, 0x28, 0xa4b3a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xabc101, 0x1d, 0xc001033370, 0x42d308, 0x30, 0xa4b3a0, 0xb85601)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xabc101, 0x1d, 0xc005bced40, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc003459bc0, 0xabc101, 0x1d, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc001033518, 0xc001c5e900, 0xabc101, 0x1d, 0x6d, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:133 +0x3a7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717440, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1023 [semacquire]:
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc001ab0690, 0x69, 0xaa0378, 0x1, 0xc005ab7c00, 0x0)
/usr/local/go/src/regexp/regexp.go:560 +0x135
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0025f43d8, 0xc002170f50, 0xc0054b62c0, 0xc0025f4308)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0025f43d8, 0xc002170ee0, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0025f50c0, 0xc002170ee0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:155 +0x7cc
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0025f50c0, 0xc002170ee0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0025f50c0, 0xc002170e00, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00070f0c0, 0xc002170e00, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00070f0c0, 0xc002170e00, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0025f50c0, 0xc002170d20, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00070f0c0, 0xc002170d20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00070f0c0, 0xc002170d20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0025f50c0, 0xc002170620, 0xc001d7ae00, 0x7fe260fedc28)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00070f0c0, 0xc002170620, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00070f0c0, 0xc002170620, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0025f50c0, 0xc002170540, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00070f0c0, 0xc002170540, 0x54b170, 0xc0026faa90)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc002170540, 0x0, 0x0, 0x0, 0x0, 0xc002170540, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc001999530, 0x0, 0x0, 0x0, 0x1300, 0xc0055e7300, 0x1112, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0055e7300, 0x1112, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000b6d9e0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00070f518, 0xc001c5ed80, 0xc001dd24c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717500, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1024 [semacquire]:
strings.genSplit(0xc004876460, 0x46, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x87, 0xc004876410, 0x46)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000f0f588, 0xaa04f9, 0x2, 0xc004876410, 0x46, 0x1, 0x16, 0xc00656cb20)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc000f0f518, 0xc0032a9ee0, 0xa6)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0027175c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1025 [runnable]:
regexp.(*inputString).step(0xc004e88080, 0x3f, 0xffffffff00000034, 0x1)
/usr/local/go/src/regexp/regexp.go:365 +0xbc
regexp.(*Regexp).tryBacktrack(0xc0001fb0e0, 0xc004e88000, 0xb886e0, 0xc004e88080, 0x7fe2000000d2, 0x3e, 0x755200)
/usr/local/go/src/regexp/backtrack.go:218 +0xb5f
regexp.(*Regexp).backtrack(0xc0001fb0e0, 0x0, 0x0, 0x0, 0xc004064820, 0x9c, 0x3e, 0x0, 0x127d380, 0x0, ...)
/usr/local/go/src/regexp/backtrack.go:353 +0x39d
regexp.(*Regexp).doExecute(0xc0001fb0e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc004064820, 0x9c, 0x0, 0x0, ...)
/usr/local/go/src/regexp/exec.go:535 +0x453
regexp.(*Regexp).doMatch(...)
/usr/local/go/src/regexp/exec.go:514
regexp.(*Regexp).MatchString(...)
/usr/local/go/src/regexp/regexp.go:507
github.com/advancedlogic/GoOse.score(0xc006804270, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/images.go:85 +0x11c
github.com/advancedlogic/GoOse.WebPageImageResolver.func1(0x0, 0xc006804270)
/home/develop/go/src/github.com/advancedlogic/GoOse/images.go:158 +0x173
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc006804240, 0xc00325d3a0, 0x3)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.WebPageImageResolver(0xc0023009a0, 0x4, 0xaa216a, 0x3, 0xaa8560)
/home/develop/go/src/github.com/advancedlogic/GoOse/images.go:124 +0xc7
github.com/advancedlogic/GoOse.WebPageResolver(0xc002d7f7a0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/images.go:174 +0x36
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:153 +0x91a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717680, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1026 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1d0221400, 0x7fe1d02214c0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001affcc0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001affcb0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000155200, 0x45c, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0008a0540, 0x29, 0xc000155200, 0x45c, 0xc000155200, 0x45c, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0008a0540, 0x29, 0xc000155200, 0x45c, 0xaa0378, 0x1, 0xc000155200, 0x45c, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717740, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1027 [semacquire]:
github.com/PuerkitoBio/goquery.newSingleSelection(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:113
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc00323df80, 0xc000709418, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x9b
github.com/advancedlogic/GoOse.(*ContentExtractor).GetTags(0xc000709518, 0xc002301260, 0xc0014f1d80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:269 +0x115
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:140 +0x505
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717800, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1028 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc001146d00, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:901 +0x820
golang.org/x/net/html.(*parser).parseCurrentToken(0xc001146d00)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc001146d00, 0xc000226150, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc004c38140, 0x0, 0x0, 0x0, 0xc003223200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc004c38140, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0032238f8, 0xc00328f000, 0x2609, 0x6576616820353220, 0x68742074736f6c20, 0x6576696c20726965)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0027178c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1029 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions.func1(0x13d, 0x137, 0x101)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:49 +0x40
sort.symMerge_func(0xc0003237d0, 0xc00594e5d0, 0x136, 0x13a, 0x140)
/usr/local/go/src/sort/zfuncversion.go:233 +0x261
sort.symMerge_func(0xc0003237d0, 0xc00594e5d0, 0x12c, 0x136, 0x140)
/usr/local/go/src/sort/zfuncversion.go:247 +0x2ee
sort.symMerge_func(0xc0003237d0, 0xc00594e5d0, 0x118, 0x12f, 0x140)
/usr/local/go/src/sort/zfuncversion.go:247 +0x2ee
sort.symMerge_func(0xc0003237d0, 0xc00594e5d0, 0xf0, 0x11b, 0x140)
/usr/local/go/src/sort/zfuncversion.go:247 +0x2ee
sort.symMerge_func(0xc0003237d0, 0xc00594e5d0, 0xa0, 0xf4, 0x140)
/usr/local/go/src/sort/zfuncversion.go:247 +0x2ee
sort.symMerge_func(0xc0003237d0, 0xc00594e5d0, 0x0, 0xa0, 0x140)
/usr/local/go/src/sort/zfuncversion.go:247 +0x2ee
sort.stable_func(0xc0003237d0, 0xc00594e5d0, 0x4d7)
/usr/local/go/src/sort/zfuncversion.go:175 +0xc2
sort.SliceStable(0x9de7e0, 0xc001001800, 0xc0003237d0)
/usr/local/go/src/sort/slice.go:27 +0xcb
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc003763000, 0xfe9, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:48 +0x1c8
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc003763000, 0xfe9, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc003763000, 0xfe9, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc003763000, 0xfe9, 0x0, 0x0, 0xfe9, 0xc003763000, 0xfe9)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc003763000, 0xfe9, 0xfe9, 0x1300)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000dadddc, 0x1b, 0xc003763000, 0xfe9, 0xc003763000, 0xfe9, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000dadddc, 0x1b, 0xc003763000, 0xfe9, 0xaa0378, 0x1, 0xc003763000, 0xfe9, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717980, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1030 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc001001620, 0xc0004312c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0004313c0, 0xaa05c9, 0x2, 0xc001842a00, 0x500, 0x5, 0x107, 0xc000763e60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001842a00, 0x500, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000431518, 0xc002300ce0, 0x53, 0xc001fffd40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717a40, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1031 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0033393c0, 0xaa04f9, 0x2, 0xc001f3f300, 0x12a6, 0xc001f3f300, 0xc0033393b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001f3f300, 0x12a6, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003339518, 0xc0050486a0, 0x2a, 0xc00112d2f0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717b00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1032 [semacquire]:
golang.org/x/net/html.NewTokenizerFragment(0xb7c280, 0xc005a64320, 0x0, 0x0, 0x20)
/home/develop/go/src/golang.org/x/net/html/token.go:1215 +0x43
golang.org/x/net/html.NewTokenizer(...)
/home/develop/go/src/golang.org/x/net/html/token.go:1201
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc005a64320, 0x0, 0x0, 0x0, 0xc003b5d200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2337 +0x41
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc005a64320, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc003b5d8f8, 0xc001ae1000, 0xf8a, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717bc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1033 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc00419ffb0, 0x13626b470, 0x1251a20, 0xc001c5ed00, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717c80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1034 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc005187410, 0x1, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords.func1(0x0, 0xc005187410)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:174 +0x4a
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc0051873e0, 0xc00354d2c8, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords(0xc00354d388)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:173 +0xb0
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc00354d388, 0xc002d3c900, 0xaa04f9, 0x2, 0x42d308, 0x8, 0x9ecc80, 0xc00354d301, 0xc003956ab8)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:43 +0xf5
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc00354d518, 0xc002d3c900, 0xaa04f9, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717d40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1035 [runnable]:
strings.Index(0xc00661aa00, 0x6f2, 0xaa0378, 0x1, 0x139)
/usr/local/go/src/strings/strings.go:1028 +0x67a
strings.genSplit(0xc00661aa00, 0x6f2, 0xaa0378, 0x1, 0x0, 0x139, 0xc05, 0xc00661a300, 0x6f2)
/usr/local/go/src/strings/strings.go:260 +0xf3
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc004a41588, 0xaa04f9, 0x2, 0xc00661a300, 0x6f2, 0x9, 0x1f, 0xc003382620)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc004a41518, 0xc0004e7ea0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717e00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1037 [semacquire]:
github.com/PuerkitoBio/goquery.pushStack(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/utilities.go:159
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc004c1e2d0, 0xaa3da6, 0x4, 0xffffffffffffffff)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0xb3
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0046d9518, 0xc002f719e0, 0x3e, 0xc00322eb80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:120 +0x4a
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002717f80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1038 [semacquire]:
regexp.(*Regexp).replaceAll(0xc004b20140, 0x0, 0x0, 0x0, 0xc002916000, 0x11f8, 0x2, 0xc0008c5218, 0xc002916000, 0x11f8, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc004b20140, 0xc002916000, 0x11f8, 0xaa0378, 0x1, 0x1, 0x27)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0011cd518, 0xc001397a20, 0xc0023016c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:319 +0x778
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00271e060, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1039 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005bcf840, 0xc0010b32c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0010b33c0, 0xaa058b, 0x2, 0xc0016ab200, 0x5ff, 0x6, 0x13e, 0xc00137f2c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0016ab200, 0x5ff, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0010b3518, 0xc001396d40, 0x30, 0xc000e7af60)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00271e120, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1040 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00137e4e0, 0xc0024232c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0024233c0, 0xaa04f9, 0x2, 0xc002f6d500, 0x99a, 0xc002f6d500, 0xc0024233b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002f6d500, 0x99a, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002423518, 0xc0000b2a00, 0x48, 0xc000ffe2d0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00271e1e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1042 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe16c025a60, 0x7fe16c025cd0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001d076b0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001d076a0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0019e6000, 0x406, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0012bcae0, 0x2d, 0xc0019e6000, 0x406, 0xc0019e6000, 0x406, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0012bcae0, 0x2d, 0xc0019e6000, 0x406, 0xaa0378, 0x1, 0xc0019e6000, 0x406, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00271e360, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1043 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc0009d7300, 0x11dc, 0xc001f4b218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc0009d7300, 0x11dc, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001f4b3c0, 0xaa04d9, 0x2, 0xc0009d7300, 0x11dc, 0x1, 0x33f, 0xc000fee380)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0009d7300, 0x11dc, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001f4b518, 0xc001885a20, 0x41, 0xc0014192c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00271e420, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1044 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc00321d900, 0xc4b, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:81 +0x208
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc00321d900, 0xc4b, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc00321d900, 0xc4b, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc00321d900, 0xc4b, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00321d900, 0xc4b, 0x0, 0x0, 0xc4b, 0xc00321d900, 0xc4b)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00321d900, 0xc4b, 0xc4b, 0xc80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002e2a800, 0x3e, 0xc00321d900, 0xc4b, 0xc00321d900, 0xc4b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002e2a800, 0x3e, 0xc00321d900, 0xc4b, 0xaa0378, 0x1, 0xc00321d900, 0xc4b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00271e4e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1045 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc00256f1e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x18a
golang.org/x/net/html.(*parser).parse(0xc00256f2b0, 0xc0043f5c00, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0010000a0, 0x0, 0x0, 0x0, 0xc003c01200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0010000a0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc003c018f8, 0xc00215e380, 0xcb2, 0xc001224898, 0x3, 0xc00122489c)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00271e5a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1046 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc000c4f860, 0x1, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords.func1(0x0, 0xc000c4f860)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:174 +0x4a
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc000c4f830, 0xc000f592c8, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*outputFormatter).removeParagraphsWithFewWords(0xc000f59388)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:173 +0xb0
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc000f59388, 0xc0038d6990, 0xaa04f9, 0x2, 0x42d308, 0x8, 0x9ecc80, 0xc000f59301, 0xc0032552b8)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:43 +0xf5
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc000f59518, 0xc0038d6990, 0xaa04f9, 0x2, 0x0, 0x0, 0x6100000020, 0x730000006c, 0x200000006f)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00271e660, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1047 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc00441aa80, 0x961, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc00441aa80, 0x961, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc00441aa80, 0x961, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc00441aa80, 0x961, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00441aa80, 0x961, 0x0, 0x0, 0x961, 0xc00441aa80, 0x961)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00441aa80, 0x961, 0x961, 0xc00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001fef044, 0x35, 0xc00441aa80, 0x961, 0xc00441aa80, 0x961, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001fef044, 0x35, 0xc00441aa80, 0x961, 0xaa0378, 0x1, 0xc00441aa80, 0x961, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00271e720, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1049 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc001c7ec30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc001c7edd0, 0xc0011ef960, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc001cfcf60, 0x0, 0x0, 0x0, 0xc001f4d200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc001cfcf60, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc001f4d8f8, 0xc00097a000, 0xf29, 0x6f00000020, 0x200000006e, 0x7200000066)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00271e8a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1051 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc002ca0340, 0xc001bf1000, 0xe47)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.textIM(0xc002ca0340, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:1381 +0xf2
golang.org/x/net/html.(*parser).parseCurrentToken(0xc002ca0340)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc002ca0340, 0xc006524070, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc001884f40, 0x0, 0x0, 0x0, 0xc004523200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc001884f40, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0045238f8, 0xc003e16000, 0x1c11, 0xc001226565, 0x3, 0xc001226569)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00271ea20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1052 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc00359a380, 0x309, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:59 +0x21d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc00359a380, 0x309, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc00359a380, 0x309, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00359a380, 0x309, 0x0, 0x0, 0x309, 0xc00359a380, 0x309)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00359a380, 0x309, 0x309, 0x380)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000764377, 0x44, 0xc00359a380, 0x309, 0xc00359a380, 0x309, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000764377, 0x44, 0xc00359a380, 0x309, 0xaa0378, 0x1, 0xc00359a380, 0x309, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00271eae0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1053 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1dc16d110, 0x7fe1dc16d2d0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0021eae70, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0021eae50, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0016f6800, 0xa8e, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0014180f0, 0x47, 0xc0016f6800, 0xa8e, 0xc0016f6800, 0xa8e, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0014180f0, 0x47, 0xc0016f6800, 0xa8e, 0xaa0378, 0x1, 0xc0016f6800, 0xa8e, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00271eba0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 408 [semacquire]:
strings.genSplit(0xc0033cb800, 0x729, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc005316000, 0x17a, 0x17a)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001f393c0, 0xaa04f9, 0x2, 0xc004f3d800, 0x729, 0x1, 0x17a, 0xc00115b800)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004f3d800, 0x729, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001f39518, 0xc004c39020, 0x3f, 0xc002ed8740)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061fb00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1109 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc00517b000, 0x786, 0xc0031bf218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc00517b000, 0x786, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0031bf3c0, 0xaa0569, 0x2, 0xc00517b000, 0x786, 0x27, 0x187, 0xc0000b35e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00517b000, 0x786, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0031bf518, 0xc0000b3140, 0x30, 0xc0006e6690)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf380, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1108 [semacquire]:
strings.genSplit(0xc004b2a000, 0x7c3, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x0, 0xc0007f1a40, 0x30000c00196b360)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00196b3c0, 0xaa04f9, 0x2, 0xc005e6b800, 0x7c3, 0xc005e6b800, 0xc00196b3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005e6b800, 0x7c3, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00196b518, 0xc003309820, 0x45, 0xc003651450)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf200, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1060 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2340a13e0, 0x7fe1d8002810)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001c409e0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001c409d0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc004f04c00, 0x385, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002b6a3c0, 0x36, 0xc004f04c00, 0x385, 0xc004f04c00, 0x385, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002b6a3c0, 0x36, 0xc004f04c00, 0x385, 0xaa0378, 0x1, 0xc004f04c00, 0x385, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001feff20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1061 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc003275600, 0xc9f, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:59 +0x21d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc003275600, 0xc9f, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc003275600, 0xc9f, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc003275600)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc003275600, 0xc9f, 0x0, 0x0, 0xc9f, 0x1000, 0xc003275600)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc003275600, 0xc9f, 0xc9f, 0x1000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0008967e0, 0x52, 0xc003275600, 0xc9f, 0xc003275600, 0xc9f, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0008967e0, 0x52, 0xc003275600, 0xc9f, 0xaa0378, 0x1, 0xc003275600, 0xc9f, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000896840, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1062 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc001cfaa00, 0x13df, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc001cfaa00, 0x13df, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc001cfaa00, 0x13df, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc001cfaa00, 0x13df, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc001cfaa00, 0x13df, 0x0, 0x0, 0x13df, 0xc001cfaa00, 0x13df)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc001cfaa00, 0x13df, 0x13df, 0x1500)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00305e2a0, 0x58, 0xc001cfaa00, 0x13df, 0xc001cfaa00, 0x13df, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00305e2a0, 0x58, 0xc001cfaa00, 0x13df, 0xaa0378, 0x1, 0xc001cfaa00, 0x13df, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000896900, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1063 [semacquire]:
github.com/gigawattio/window.Rolling(0xc0034fb300, 0x195, 0x195, 0x3, 0x0, 0x194, 0xc0034fb300)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002f4b518, 0xc004c38f40, 0xc00135e480)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000896c00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1064 [semacquire]:
strings.genSplit(0xc0052e0000, 0x934, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc005316000, 0x1d6, 0x1d6)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0044113c0, 0xaa04fd, 0x2, 0xc00182b500, 0x934, 0x1, 0x1d6, 0xc000762220)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00182b500, 0x934, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004411518, 0xc001dd26e0, 0x34, 0xc0023e2946)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000896cc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1065 [semacquire]:
bytes.makeSlice(0x2971, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0042dd388, 0xff, 0x1)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0042dd388, 0xc005be4500, 0xff, 0x1, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc005e8efc0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc005e8ee00)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc005bcd490)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc005bcd3b0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc005bcccb0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc001914750, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0042dd518, 0xc00608c320, 0xd, 0xc0010dafc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000896d80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1066 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005049d40, 0xc00269f2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00269f3c0, 0xaa04f9, 0x2, 0xc00508e000, 0xc84, 0x9, 0x2ab, 0xc0004e68a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00508e000, 0xc84, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00269f518, 0xc001884940, 0x34, 0xc000b72900)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000896f00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1067 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc004244000, 0x3a69, 0xc004475218, 0xc001d7aa80)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc004244000, 0x3a69, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0044753c0, 0xaa04f9, 0x2, 0xc004244000, 0x3a69, 0x1, 0xb06, 0xc00642bbe0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004244000, 0x3a69, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004475518, 0xc0034d1dc0, 0x4d, 0xc000fff090)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000896fc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1068 [semacquire]:
strings.genSplit(0xc00582c900, 0x85c, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc000f46000, 0x1bd, 0x1bd)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002abd3c0, 0xaa04d9, 0x2, 0xc002d91b00, 0x85c, 0x1, 0x1bd, 0xc00642a580)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002d91b00, 0x85c, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002abd518, 0xc003213980, 0x52, 0xc003d618c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000897080, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1069 [semacquire]:
bytes.makeSlice(0x600, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc003a050e8, 0x200, 0x200)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).ReadFrom(0xc003a050e8, 0xb7b620, 0xc0041f30e0, 0x203001, 0x203001, 0x203001)
/usr/local/go/src/bytes/buffer.go:202 +0x48
io/ioutil.readAll(0xb7b620, 0xc0041f30e0, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:36 +0xe3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc0041f30e0, 0xc003a051d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0041f30e0, 0x0, 0x0, 0x0, 0xa80, 0xc005d26000, 0x918, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc005d26000, 0x918, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc002eea6e0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003a05518, 0xc002f702e0, 0xc005835be0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000897320, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1070 [semacquire]:
regexp/syntax.Parse(0xaa5b70, 0x5, 0xc0022c00d4, 0x7fe260fed501, 0x47f290, 0xc00224af00)
/usr/local/go/src/regexp/syntax/parse.go:712 +0x54
regexp.compile(0xaa5b70, 0x5, 0xd4, 0x4, 0xc0022c7248, 0x509d68)
/usr/local/go/src/regexp/regexp.go:170 +0x5a
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
regexp.MustCompile(0xaa5b70, 0x5, 0xaa0378)
/usr/local/go/src/regexp/regexp.go:309 +0x4b
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002f79518, 0xc001dd3300, 0xc001dd3c60)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:324 +0x7f8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0008973e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1071 [semacquire]:
strings.genSplit(0xc003715800, 0x7c5, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xe97, 0xc0056d8000, 0x7c5)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00514f588, 0xaa04f9, 0x2, 0xc0056d8000, 0x7c5, 0x0, 0xc003be6a80, 0x7fe260fee2f0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00514f518, 0xc002408800, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0008974a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1072 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002fb1588, 0xaa04f9, 0x2, 0xc005d80000, 0x1cd, 0x13, 0xc002fb1290, 0x42d308)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0xe, 0xc005312570)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:635 +0x82
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc00037d350, 0xc002fb1320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc002fb1518, 0xc005c8e3c0, 0xc002409040)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc002fb1518, 0xc005c8e3c0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc002fb1518, 0xc005c8e3c0, 0xc005c8e3c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000897560, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1073 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe134004040, 0x7fe1340040e0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0014f5c00, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0014f5bf0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0024dea00, 0x6fd, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000b6c030, 0x18, 0xc0024dea00, 0x6fd, 0xc0024dea00, 0x6fd, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000b6c030, 0x18, 0xc0024dea00, 0x6fd, 0xaa0378, 0x1, 0xc0024dea00, 0x6fd, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000897620, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1074 [semacquire]:
golang.org/x/net/html.NewTokenizerFragment(0xb7c280, 0xc0033833e0, 0x0, 0x0, 0x20)
/home/develop/go/src/golang.org/x/net/html/token.go:1215 +0x43
golang.org/x/net/html.NewTokenizer(...)
/home/develop/go/src/golang.org/x/net/html/token.go:1201
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0033833e0, 0x0, 0x0, 0x0, 0xc001783200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2337 +0x41
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0033833e0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0017838f8, 0xc0039c2000, 0xde1, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0008976e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1075 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1f0231db0, 0x7fe1f0231f10)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0015fd460, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0015fd440, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001bd1000, 0x7d9, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002292087, 0x30, 0xc001bd1000, 0x7d9, 0xc001bd1000, 0x7d9, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002292087, 0x30, 0xc001bd1000, 0x7d9, 0xaa0378, 0x1, 0xc001bd1000, 0x7d9, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0008977a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1076 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc003dc5100, 0x6bd, 0xc003559218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc003dc5100, 0x6bd, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0035593c0, 0xaa0599, 0x2, 0xc003dc5100, 0x6bd, 0x1, 0x161, 0xc00656daa0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003dc5100, 0x6bd, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003559518, 0xc002f70bc0, 0x34, 0xc003493100)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000897860, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1077 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0051610c0, 0xc003210b40, 0xe6, 0xe6, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:430 +0x233
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0051610c0, 0xc002357180, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0051610c0, 0xc002356fc0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0051610c0, 0xc002356fc0, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0051610c0, 0xc002356fc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0051610c0, 0xc002356fc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0051610c0, 0xc002356850, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0017890c0, 0xc002356850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0017890c0, 0xc002356850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0051610c0, 0xc002356770, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0017890c0, 0xc002356770, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0017890c0, 0xc002356770, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0051610c0, 0xc00120fea0, 0xc0039fea80, 0x7fe260fef080)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0017890c0, 0xc00120fea0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0017890c0, 0xc00120fea0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0051610c0, 0xc00120fdc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0017890c0, 0xc00120fdc0, 0x54b170, 0xc002eed110)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc00120fdc0, 0x0, 0x0, 0x0, 0x0, 0xc00120fdc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0047db260, 0x0, 0x0, 0x0, 0x900, 0xc005182400, 0x8f5, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc005182400, 0x8f5, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000cd5a40)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001789518, 0xc002408aa0, 0xc000763540)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000897920, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1078 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc004229110, 0xc4, 0xc4, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc004229110, 0xc4, 0x7544a2, 0x0)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000b0b588, 0xaa04f9, 0x2, 0xc004229110, 0xc4, 0x21, 0x129, 0xc0034d01e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc000b0b518, 0xc002f70fe0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0008979e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1079 [semacquire]:
regexp.(*Regexp).replaceAll(0xc000149540, 0x0, 0x0, 0x0, 0xc001b33b80, 0x28f, 0x2, 0xc005257038, 0xc001b33b80, 0x28f, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc000149540, 0xc001b33b80, 0x28f, 0xaa04bf, 0x2, 0x1, 0x1)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.FromHTMLNode(0xc00524e2a0, 0x0, 0x0, 0x0, 0x0, 0xc00524e2a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:80 +0x191
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0047db9b0, 0x0, 0x0, 0x0, 0x580, 0xc0018a1600, 0x562, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0018a1600, 0x562, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000c53890)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002f77518, 0xc002f70c20, 0xc000763b40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000897b60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1080 [semacquire]:
strings.genSplit(0xc002ca3800, 0x5fa, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc002ae4a00, 0x13c, 0x13c)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0030513c0, 0xaa04fd, 0x2, 0xc005225200, 0x5fa, 0x1, 0x13c, 0xc00656d7c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005225200, 0x5fa, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003051518, 0xc000b33d40, 0x38, 0xc001bfbf40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000897f80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1082 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc002f62a80, 0x92c, 0xc003a03218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc002f62a80, 0x92c, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003a033c0, 0xaa04fd, 0x2, 0xc002f62a80, 0x92c, 0x1, 0x1d5, 0xc00229cfa0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002f62a80, 0x92c, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003a03518, 0xc00522c120, 0x2a, 0xc000c53620)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427020, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1083 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0011e7400, 0x3e0, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:59 +0x21d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0011e7400, 0x3e0, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0011e7400, 0x3e0, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0011e7400, 0x3e0, 0x0, 0x0, 0x3e0, 0xc0011e7400, 0x3e0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0011e7400, 0x3e0, 0x3e0, 0x480)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002176140, 0x3f, 0xc0011e7400, 0x3e0, 0xc0011e7400, 0x3e0, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002176140, 0x3f, 0xc0011e7400, 0x3e0, 0xaa0378, 0x1, 0xc0011e7400, 0x3e0, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004271a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1084 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc005a62360, 0x12da3c864, 0x1251a20, 0xc005a64100, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427320, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1085 [runnable]:
strings.Index(0xc002b7ac00, 0x12f6, 0xaa0378, 0x1, 0x3ae)
/usr/local/go/src/strings/strings.go:1028 +0x67a
strings.genSplit(0xc002b7ac00, 0x12f6, 0xaa0378, 0x1, 0x0, 0x3ae, 0xc006960000, 0x3af, 0x3af)
/usr/local/go/src/strings/strings.go:260 +0xf3
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002f733c0, 0xaa0599, 0x2, 0xc003091900, 0x12f6, 0x1, 0x3af, 0xc000762160)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003091900, 0x12f6, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002f73518, 0xc00115a4c0, 0x40, 0xc00235ca80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004274a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1087 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc000593320, 0x14befdcd6, 0x1251a20, 0xc001cfc260, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004277a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1088 [runnable]:
regexp.(*Regexp).ReplaceAllString.func1(0xc0067b6000, 0x288, 0x500, 0xc0042c4a00, 0x2, 0x2, 0x236, 0x500, 0x251)
/usr/local/go/src/regexp/regexp.go:557 +0xfe
regexp.(*Regexp).replaceAll(0xc000149ae0, 0x0, 0x0, 0x0, 0xc003b68002, 0x6d4, 0x2, 0xc003a5ba58, 0xaa3a2a, 0x4, ...)
/usr/local/go/src/regexp/regexp.go:616 +0x2e9
regexp.(*Regexp).ReplaceAllString(0xc000149ae0, 0xc003b68002, 0x6d4, 0x0, 0x0, 0xc00038e060, 0x127d7a0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
_/home/ec2-user/environment/cluster2/cluster2/base.Sanitize(0xaa3a2a, 0x4, 0xc003b68002, 0x6d4, 0x0, 0x0, 0x10, 0xab52f4)
/home/ec2-user/environment/cluster2/cluster2/base/sanitizer.go:44 +0xc9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427920, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:304 +0x33a
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1089 [runnable]:
github.com/fatih/set.(*Set).Add(0xc000fefdc0, 0xc003a932c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003a933c0, 0xaa05c9, 0x2, 0xc001088000, 0xfb3, 0x3, 0x300, 0xc004cda2a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001088000, 0xfb3, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003a93518, 0xc0050494a0, 0x5d, 0xc0018f4269)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007e180, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1090 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1fc223aa0, 0x7fe21c002b60)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc001c1a610, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc001c1a5f0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003b68700, 0x66e, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0011263c0, 0x4a, 0xc003b68700, 0x66e, 0xc003b68700, 0x66e, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0011263c0, 0x4a, 0xc003b68700, 0x66e, 0xaa0378, 0x1, 0xc003b68700, 0x66e, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007e480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1091 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc00506f500, 0xc00506f500, 0xc00506f600)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc003e27518, 0xc001cfdba0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:423 +0x38b
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007e600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1092 [runnable]:
github.com/fatih/set.(*Set).Add(0xc000c446e0, 0xc0025a7220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:26 +0x124
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0025a7588, 0xaa04f9, 0x2, 0xc000721e40, 0x145, 0x1, 0x142, 0xc000fefc60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0025a7518, 0xc0010b08c0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f5c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1094 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe22c006100, 0x7fe22c0027b0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0038cfa10, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0038cfa00, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc00380a000, 0x3ea, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000bbbbd0, 0x4b, 0xc00380a000, 0x3ea, 0xc00380a000, 0x3ea, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000bbbbd0, 0x4b, 0xc00380a000, 0x3ea, 0xaa0378, 0x1, 0xc00380a000, 0x3ea, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f980, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1095 [semacquire]:
regexp.makeOnePass(0xc00568a7e0, 0xc00568a7e0)
/usr/local/go/src/regexp/onepass.go:309 +0x20e
regexp.compileOnePass(0xc001283470, 0xc001283470)
/usr/local/go/src/regexp/onepass.go:500 +0x129
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0001caff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0001caff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc0023d82a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0001caff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0001caff8, 0x1, 0xc001926d20, 0xc0100081501, 0x280460fedc28)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0001caff8, 0x4, 0xb40, 0xc004295401, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc004295000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc0022b1450, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc001283410, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0044938f8, 0xc001cf8000, 0x14b4, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007fbc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1096 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc003951fb0, 0xc003951fb0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc005195518, 0xc0032a8ac0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:423 +0x38b
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007fd40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1097 [semacquire]:
strings.genSplit(0xc00237e000, 0xffd, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc005c60000, 0x32b, 0x32b)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0008553c0, 0xaa059b, 0x2, 0xc00452d000, 0xffd, 0x17, 0x32b, 0xc005c54060)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00452d000, 0xffd, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000855518, 0xc00115afe0, 0x38, 0xc001bfabc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74060, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1098 [semacquire]:
bytes.makeSlice(0x2da, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00196d388, 0x152, 0x1)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00196d388, 0xc0024e29a0, 0x152, 0x1, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0049ecd20)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0049ecb60)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0049ec540)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0049ec460)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00232e150)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc005032960, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00196d518, 0xc0010b14c0, 0x44, 0xc004642af0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74420, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1099 [semacquire]:
regexp.(*Regexp).replaceAll(0xc000149540, 0x0, 0x0, 0x0, 0xc004549600, 0xd31, 0x2, 0xc005245038, 0xc004549600, 0xd31, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc000149540, 0xc004549600, 0xd31, 0xaa04bf, 0x2, 0x1, 0x2)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.FromHTMLNode(0xc00080ae70, 0x0, 0x0, 0x0, 0x0, 0xc00080ae70, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:80 +0x191
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0048d3a70, 0x0, 0x0, 0x0, 0x1000, 0xc0019d8000, 0xe69, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0019d8000, 0xe69, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000115470)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc005245518, 0xc0009c69e0, 0xc0009c7780)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f745a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1100 [semacquire]:
github.com/fatih/set.newTS(...)
/home/develop/go/src/github.com/fatih/set/set_ts.go:15
github.com/fatih/set.New(...)
/home/develop/go/src/github.com/fatih/set/set.go:56
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0019c3588, 0xaa04f9, 0x2, 0xc00580e000, 0x168, 0x7, 0xc0019c3290, 0x42d308)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:74 +0x52
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0x3, 0xc000aa7a40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:635 +0x82
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc000aa7740, 0xc0019c3320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc0019c3518, 0xc000c46720, 0xc005d7b360)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc0019c3518, 0xc000c46720, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc0019c3518, 0xc000c46720, 0xc000c46720)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74720, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1101 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1b428f660, 0x7fe150014850)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc002d7bfd0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc002d7bfc0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc000afe480, 0x43e, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000bbb810, 0x45, 0xc000afe480, 0x43e, 0xc000afe480, 0x43e, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000bbb810, 0x45, 0xc000afe480, 0x43e, 0xaa0378, 0x1, 0xc000afe480, 0x43e, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f748a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1102 [semacquire]:
bytes.makeSlice(0x16fb, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00143eff0, 0x1, 0xc000142000)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteByte(0xc00143eff0, 0xc00002033e, 0xad0185, 0x1)
/usr/local/go/src/bytes/buffer.go:267 +0x8c
golang.org/x/net/html.render1(0xb84d20, 0xc00143eff0, 0xc0007dd110, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:183 +0x56c
golang.org/x/net/html.render1(0xb84d20, 0xc00143eff0, 0xc0007dd030, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00143eff0, 0xc0007dbd50, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00143eff0, 0xc0007dbc70, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00143eff0, 0xc002a2e150, 0xb84d20, 0xc00143eff0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc00143eff0, 0xc002a2e150, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc00143ee40, 0xd6b2a8c0, 0x1251a20, 0xc0000b24e0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74a20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1103 [semacquire]:
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0009551c0, 0x2649d0b00000000, 0xc000955190, 0xc000955218, 0x42ca96, 0xc00416e750)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:824 +0x69
github.com/andybalholm/cascadia.ParseGroup(0xab7d92, 0x15, 0x1, 0x1, 0xc000955230, 0xc000955248, 0x42d308)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xab7d92, 0x15, 0xc00416e750, 0x1, 0xb856e0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc001291620, 0xab7d92, 0x15, 0xc00416e750)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*Cleaner).removeScriptsStyle(0xc0009556b0, 0xc00000f420, 0xc00000f420)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:407 +0x5f
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc0009556b0, 0xc00000f420, 0xc00522c000)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:292 +0xc6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f74ba0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1110 [semacquire]:
github.com/PuerkitoBio/goquery.findWithMatcher.func1(0x0, 0xc0004dc2a0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:547 +0x180
github.com/PuerkitoBio/goquery.mapNodes(0xc000011838, 0x1, 0x1, 0xc0022bb0b0, 0x1, 0x0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:693 +0xf6
github.com/PuerkitoBio/goquery.findWithMatcher(0xc000011838, 0x1, 0x1, 0xb856e0, 0xc00656d9a0, 0xc001b63fb0, 0xc001b63fb0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:543 +0x81
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc00143ff50, 0xaa408e, 0x4, 0xaa04f9)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x85
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:63 +0xd5
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0022bb8f8, 0xc002847500, 0x9df, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf500, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1111 [semacquire]:
bytes.makeSlice(0x1860, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc001a42d80, 0x86, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc001a42d80, 0xc000b69710, 0x86, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc001a42d80, 0xc000b69710, 0x86, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc001a42d80, 0xc002a350a0, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc001a42d80, 0xc002a35030, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001a42d80, 0xc002283b90, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001a42d80, 0xc002283ab0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001a42d80, 0xc0022833b0, 0xb84d20, 0xc001a42d80)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc001a42d80, 0xc0022833b0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc001a42bd0, 0x1748015ca, 0x1251a20, 0xc003309ee0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf740, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1112 [semacquire]:
regexp.onePassCopy(0xc001d080f0, 0xc001d080f0)
/usr/local/go/src/regexp/onepass.go:222 +0x77
regexp.compileOnePass(0xc001d080f0, 0xc001d080f0)
/usr/local/go/src/regexp/onepass.go:497 +0x11b
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0022a8ff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0022a8ff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc003a80698)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0022a8ff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0022a8ff8, 0x1, 0xc0013ae9c0, 0xc0100500700, 0x280460fee2f0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0022a8ff8, 0x4, 0x715, 0xc001d95801, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc001d95000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc001bdb380, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc003738150, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0022a98f8, 0xc0036db600, 0x8e5, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcf8c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1113 [semacquire]:
strings.genSplit(0xc0063bd180, 0x555, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc002440c00, 0x127, 0x127)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00314f3c0, 0xaa05c9, 0x2, 0xc0063bbb80, 0x555, 0x5, 0x127, 0xc005a64480)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0063bbb80, 0x555, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00314f518, 0xc003309ce0, 0x3d, 0xc0046af300)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcfa40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1114 [semacquire]:
regexp.makeOnePass.func1(0x6, 0xc003350600, 0x10, 0x10, 0x1)
/usr/local/go/src/regexp/onepass.go:385 +0xbdf
regexp.makeOnePass(0xc000cca060, 0xc000cca060)
/usr/local/go/src/regexp/onepass.go:448 +0x398
regexp.compileOnePass(0xc002c37f50, 0xc002c37f50)
/usr/local/go/src/regexp/onepass.go:500 +0x129
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc000192ff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc000192ff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc002b619bc)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc000192ff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc000192ff8, 0x1, 0xc0005710a0, 0xc0103df9c00, 0x280460fed560)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc000192ff8, 0x4, 0x5f6, 0xc004d0fa01, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc004d0f000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc000109ee0, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc002c37ec0, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc001f3d8f8, 0xc001d10000, 0xa06, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcfbc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1115 [semacquire]:
strings.genSplit(0xc0041a0900, 0x809, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc006384000, 0x1ad, 0x1ad)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0008753c0, 0xaa0569, 0x2, 0xc000a4c900, 0x809, 0x3c, 0x1ad, 0xc001001460)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000a4c900, 0x809, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000875518, 0xc001001060, 0x48, 0xc000ea7c20)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcfd40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1116 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc003d5c1a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc003d5c270, 0xc0006b5500, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc002226100, 0x0, 0x0, 0x0, 0xc00371b200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc002226100, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00371b8f8, 0xc002994800, 0x1630, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fcfec0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1117 [runnable]:
github.com/fatih/set.(*Set).Has(0xc00039e200, 0xc00371d2c0, 0x1, 0x1, 0xc000480000)
/home/develop/go/src/github.com/fatih/set/set_ts.go:71 +0x169
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00371d3c0, 0xaa04d9, 0x2, 0xc004327600, 0x57f, 0x1, 0x133, 0xc004cdaf60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:81 +0x1e7
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004327600, 0x57f, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00371d518, 0xc002226300, 0x46, 0xc0036517c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e1e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1118 [semacquire]:
bytes.makeSlice(0xace, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc003b363c0, 0x8, 0xc002b22dd8)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc003b363c0, 0xc003516480, 0x8, 0x6, 0xffffffffffffffff, 0xad0190)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc003b363c0, 0xc003516480, 0x8, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc003b363c0, 0xc0042765b0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc003b363c0, 0xc000aedf80, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc003b363c0, 0xc000aedea0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc003b363c0, 0xc000aed7a0, 0xb84d20, 0xc003b363c0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc003b363c0, 0xc000aed7a0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc004d5c9c0, 0x19426f4d5, 0x1251a20, 0xc00229c6e0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e360, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1119 [semacquire]:
strings.genSplit(0xc003be8a80, 0x923, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x0, 0xc000000000, 0x3007fe238301228)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0037433c0, 0xaa04f9, 0x2, 0xc00521e000, 0x923, 0xc00521e000, 0xc0037433b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00521e000, 0x923, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003743518, 0xc0000b3ba0, 0x42, 0xc0036519f0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e4e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1120 [semacquire]:
github.com/gigawattio/window.Rolling(0xc004661800, 0x166, 0x166, 0x3, 0x0, 0x165, 0xc004661800)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00196f518, 0xc002f700a0, 0xc002f718c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e660, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1121 [semacquire]:
github.com/gigawattio/window.Rolling(0xc00243d300, 0x115, 0x115, 0x3, 0x0, 0x114, 0xc00243d300)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:10 +0x6c
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00196f518, 0xc0000b2fe0, 0xc0000b3ca0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e7e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1138 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc00437e000, 0x914, 0xc003203218, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc00437e000, 0x914, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0032033c0, 0xaa0599, 0x2, 0xc00437e000, 0x914, 0x1, 0x1b7, 0xc00642a560)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00437e000, 0x914, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003203518, 0xc004c388a0, 0x1a, 0xc000b5ca80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045e960, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1139 [semacquire]:
bytes.makeSlice(0x600, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc001aa70e8, 0x200, 0x200)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).ReadFrom(0xc001aa70e8, 0xb7b620, 0xc0039890b0, 0x203000, 0x203000, 0x203000)
/usr/local/go/src/bytes/buffer.go:202 +0x48
io/ioutil.readAll(0xb7b620, 0xc0039890b0, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:36 +0xe3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc0039890b0, 0xc001aa71d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0039890b0, 0x0, 0x0, 0x0, 0x1300, 0xc001a3d900, 0x10cc, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc001a3d900, 0x10cc, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc0001153e0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001aa7518, 0xc0009c6440, 0xc0032a92e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045ed20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1141 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc00143f200, 0xd6b9b11f, 0x1251a20, 0xc0000b28c0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00045fb60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1142 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc004ca0000, 0x7c1, 0xc004137218, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc004ca0000, 0x7c1, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0041373c0, 0xaa04d9, 0x2, 0xc004ca0000, 0x7c1, 0x1, 0x191, 0xc00137e780)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004ca0000, 0x7c1, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004137518, 0xc003309b60, 0x35, 0xc0046aee40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fae060, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1143 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc001bdaf70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc001bdb040, 0xc0045bd880, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc002f71d80, 0x0, 0x0, 0x0, 0xc0014c3200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc002f71d80, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0014c38f8, 0xc001820600, 0x18a9, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fae660, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1144 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc0001096c0, 0xc0033515f8, 0x8)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc0001096c0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0001096c0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0001096c0, 0xc0006b21c0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc003309d40, 0x0, 0x0, 0x0, 0xc003201200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc003309d40, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0032018f8, 0xc002aec800, 0x1539, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faf020, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1145 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0055b5000, 0x16ce, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:71 +0x129
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0055b5000, 0x16ce, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0055b5000, 0x16ce, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0055b5000, 0x16ce, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0055b5000, 0x16ce, 0x0, 0x0, 0x16ce, 0xc0055b5000, 0x16ce)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0055b5000, 0x16ce, 0x16ce, 0x1a80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000116180, 0x3c, 0xc0055b5000, 0x16ce, 0xc0055b5000, 0x16ce, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000116180, 0x3c, 0xc0055b5000, 0x16ce, 0xaa0378, 0x1, 0xc0055b5000, 0x16ce, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faf920, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1146 [semacquire]:
golang.org/x/net/html.(*Tokenizer).readTag(0xc002872b60, 0x54e501)
/home/develop/go/src/golang.org/x/net/html/token.go:834 +0x162
golang.org/x/net/html.(*Tokenizer).readStartTag(0xc002872b60, 0x69)
/home/develop/go/src/golang.org/x/net/html/token.go:779 +0x45
golang.org/x/net/html.(*Tokenizer).Next(0xc002872b60, 0x1)
/home/develop/go/src/golang.org/x/net/html/token.go:1021 +0x28c
golang.org/x/net/html.(*parser).parse(0xc002872c30, 0xc003852310, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2284 +0xad
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00229cd80, 0x0, 0x0, 0x0, 0xc0036df200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00229cd80, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0036df8f8, 0xc002bd0800, 0x1539, 0xc000f5e540, 0x3, 0x3)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fafaa0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1147 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc003738270, 0x17167ad8b, 0x1251a20, 0xc0007633a0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fafc20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1148 [semacquire]:
bytes.makeSlice(0x59c, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00371f388, 0xb8, 0x5d)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00371f388, 0xc000d75b00, 0xb8, 0x5d, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000951490)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0009512d0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000950b60)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000950a80)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000950380)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc003f6c390, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00371f518, 0xc004cdab20, 0x3e, 0xc0046af940)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fafda0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1149 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0049d0660, 0x14f275902, 0x1251a20, 0xc002226260, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001faff20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1150 [semacquire]:
bytes.makeSlice(0xcbf, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc004d5de00, 0x2f, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc004d5de00, 0xc000e7b140, 0x2f, 0x6, 0xffffffffffffffff, 0xaa042b)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc004d5de00, 0xc000e7b140, 0x2f, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc004d5de00, 0xc00427cd90, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:169 +0x3a3
golang.org/x/net/html.render1(0xb84d20, 0xc004d5de00, 0xc00427cb60, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc004d5de00, 0xc000af0cb0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc004d5de00, 0xc000af0bd0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc004d5de00, 0xc000af04d0, 0xb84d20, 0xc004d5de00)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc004d5de00, 0xc000af04d0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0011aa900, 0x14f6093a7, 0x1251a20, 0xc0010b10a0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00043e840, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1151 [semacquire]:
bytes.makeSlice(0x180, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0010989c0, 0x78, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0010989c0, 0xc002d0bb00, 0x78, 0x6, 0xffffffffffffffff, 0xaa042b)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc0010989c0, 0xc002d0bb00, 0x78, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc0010989c0, 0xc000af1030, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:169 +0x3a3
golang.org/x/net/html.render1(0xb84d20, 0xc0010989c0, 0xc000af0e70, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc0010989c0, 0xc000af0e00, 0xb84d20, 0xc0010989c0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc0010989c0, 0xc000af0e00, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0011aaab0, 0x19465b79e, 0x1251a20, 0xc0010b1120, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001feeae0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1152 [semacquire]:
regexp.makeOnePass(0xc003dae720, 0xc003dae720)
/usr/local/go/src/regexp/onepass.go:309 +0x20e
regexp.compileOnePass(0xc003dae6c0, 0xc003dae6c0)
/usr/local/go/src/regexp/onepass.go:500 +0x129
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0036dcff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0036dcff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc00403e7dc)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0036dcff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0036dcff8, 0x1, 0xc000fb1e80, 0xc001d7aa80, 0x280460fee9b8)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0036dcff8, 0x4, 0x2bf, 0xc0049f2d01, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc0049f2000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc0052f21a0, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc003dae690, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0036dd8f8, 0xc00429a880, 0xd3d, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001feec60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1153 [semacquire]:
bytes.makeSlice(0x18f7, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc001098390, 0x73, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc001098390, 0xc000ad2d80, 0x73, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc001098390, 0xc000ad2d80, 0x73, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc001098390, 0xc0052db420, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc001098390, 0xc0052db3b0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001098390, 0xc000b75b20, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001098390, 0xc000b75a40, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001098390, 0xc000b75340, 0xb84d20, 0xc001098390)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc001098390, 0xc000b75340, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc001098060, 0x19448ebc3, 0x1251a20, 0xc005a657c0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef320, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1154 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc00231a380, 0x37e, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc00231a380, 0x37e, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc00231a380, 0x37e, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc00231a380, 0x37e, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00231a380, 0x37e, 0x0, 0x0, 0x37e, 0xc00231a380, 0x37e)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00231a380, 0x37e, 0x37e, 0x500)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000607810, 0x41, 0xc00231a380, 0x37e, 0xc00231a380, 0x37e, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000607810, 0x41, 0xc00231a380, 0x37e, 0xaa0378, 0x1, 0xc00231a380, 0x37e, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef6e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1155 [semacquire]:
bytes.makeSlice(0x81f, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0052c10c0, 0x1, 0x1)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0052c10c0, 0x1189d00, 0x1, 0x1, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0052c10c0, 0xc002ae7cb0, 0x82, 0x82, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:440 +0xf7
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0052c10c0, 0xc00239e9a0, 0xc0052c01b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052c10c0, 0xc001dcf8f0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0052c10c0, 0xc001dcf8f0, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0052c10c0, 0xc001dcf8f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0052c10c0, 0xc001dcf8f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052c10c0, 0xc001dced90, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00196b0c0, 0xc001dced90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00196b0c0, 0xc001dced90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052c10c0, 0xc001dcecb0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00196b0c0, 0xc001dcecb0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00196b0c0, 0xc001dcecb0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052c10c0, 0xc001dce150, 0x7fe23814da58, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00196b0c0, 0xc001dce150, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00196b0c0, 0xc001dce150, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0052c10c0, 0xc001dce070, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00196b0c0, 0xc001dce070, 0x54b170, 0xc000237e10)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc001dce070, 0x0, 0x0, 0x0, 0x0, 0xc001dce070, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0014908a0, 0x0, 0x0, 0x0, 0xd80, 0xc002f16000, 0xd2f, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc002f16000, 0xd2f, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000115830)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00196b518, 0xc0009c7880, 0xc000fee4c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fef9e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1156 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc003d5c5b0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:901 +0x820
golang.org/x/net/html.(*parser).parseCurrentToken(0xc003d5c5b0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc003d5c5b0, 0xc000951880, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc002226180, 0x0, 0x0, 0x0, 0xc0011cf200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc002226180, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0011cf8f8, 0xc001bf2a80, 0xa10, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001fefce0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1157 [semacquire]:
strings.genSplit(0xc0050da000, 0x767, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x0, 0xc000999c70, 0xc0042df360)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0042df3c0, 0xaa04fd, 0x2, 0xc004b47800, 0x767, 0xc004b47800, 0xc0042df3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004b47800, 0x767, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0042df518, 0xc002226200, 0x38, 0xc0046afa80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007e1e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1158 [semacquire]:
github.com/PuerkitoBio/goquery.newSingleSelection(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:113
github.com/PuerkitoBio/goquery.(*Selection).EachWithBreak(0xc000ccbfb0, 0xae55a0, 0x4)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:21 +0x83
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:63 +0xef
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0007ab8f8, 0xc000ef4c80, 0xc2d, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007ef00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1159 [semacquire]:
strings.Replace(0xc00132aa00, 0x32f9, 0xaa582d, 0x5, 0xaa7dce, 0x6, 0x8, 0xc00132aa00, 0x32f9)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:178 +0x153
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00305b8f8, 0xc001327500, 0x32e7, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f2c0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1160 [runnable]:
regexp.(*Regexp).tryBacktrack(0xc000149400, 0xc004e89970, 0xb886e0, 0xc004e899f0, 0xc000000001, 0x4d7, 0x0)
/usr/local/go/src/regexp/backtrack.go:209 +0x988
regexp.(*Regexp).backtrack(0xc000149400, 0x0, 0x0, 0x0, 0xc000547a80, 0x1a37, 0x4d7, 0x2, 0xc001296b80, 0x0, ...)
/usr/local/go/src/regexp/backtrack.go:353 +0x39d
regexp.(*Regexp).doExecute(0xc000149400, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc000547a80, 0x1a37, 0x4d4, 0x2, ...)
/usr/local/go/src/regexp/exec.go:535 +0x453
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc000547a80, 0x1a37, 0x2, 0xc002761e68, 0xc002761f30, 0xc002761f08, ...)
/usr/local/go/src/regexp/regexp.go:599 +0x1a5
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc000547a80, 0x1a37, 0xaa0378, 0x1, 0xc005ab7e80, 0x5)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0027630c0, 0xc005106310, 0xc004e89970, 0xc002761fa0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0027630c0, 0xc0051062a0, 0xc000000001, 0x9)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0027630c0, 0xc0051062a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0027630c0, 0xc0051062a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0027630c0, 0xc0051061c0, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0027630c0, 0xc0051061c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0027630c0, 0xc0051061c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0027630c0, 0xc0009b8fc0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0031870c0, 0xc0009b8fc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0031870c0, 0xc0009b8fc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0027630c0, 0xc0009b8ee0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0031870c0, 0xc0009b8ee0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0031870c0, 0xc0009b8ee0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0027630c0, 0xc0009b87e0, 0xc003186d40, 0x5b00000046a81e)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0031870c0, 0xc0009b87e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0031870c0, 0xc0009b87e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0027630c0, 0xc0009b8700, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0031870c0, 0xc0009b8700, 0x54b170, 0xc002ca1d40)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc0009b8700, 0x0, 0x0, 0x0, 0x0, 0xc0009b8700, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc00566b920, 0x0, 0x0, 0x0, 0x2000, 0xc0011dc000, 0x1fdd, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0011dc000, 0x1fdd, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc0033963f0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003187518, 0xc0027ee7e0, 0xc002488bc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007f8c0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1161 [runnable]:
golang.org/x/net/html.(*Tokenizer).Token(0xc000642ea0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x18a
golang.org/x/net/html.(*parser).parse(0xc000642f70, 0xc005362d20, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00642ab60, 0x0, 0x0, 0x0, 0xc000e2f200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00642ab60, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc000e2f8f8, 0xc00293e600, 0x25de, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00007fc80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1162 [runnable]:
golang.org/x/net/html.(*parser).parseCurrentToken(0xc000643110)
/home/develop/go/src/golang.org/x/net/html/parse.go:2255 +0xa0
golang.org/x/net/html.(*parser).parse(0xc000643110, 0xc002b0afc0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00642aba0, 0x0, 0x0, 0x0, 0xc0042e3200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00642aba0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0042e38f8, 0xc00421e800, 0x2764, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1163 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc003ed0b60, 0xc003351cb0, 0x8)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc003ed0b60, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc003ed0b60)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc003ed0b60, 0xc000b43c00, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc002226720, 0x0, 0x0, 0x0, 0xc00373f200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc002226720, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00373f8f8, 0xc0022ea800, 0x17e0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1164 [semacquire]:
strings.genSplit(0xc003ad7500, 0xa19, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0000c7800, 0x27c, 0x27c)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0008b73c0, 0xaa058b, 0x2, 0xc003c3f500, 0xa19, 0x5, 0x27c, 0xc005a65c40)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003c3f500, 0xa19, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0008b7518, 0xc002226820, 0x50, 0xc003651c70)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000376cc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1165 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00039fe40, 0xc0027c92c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0027c93c0, 0xaa04fd, 0x2, 0xc00330f800, 0x7f1, 0x1, 0x1ac, 0xc0059f4e00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00330f800, 0x7f1, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0027c9518, 0xc0059f49e0, 0x4f, 0xc0016a9270)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377500, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1166 [semacquire]:
strings.genSplit(0xc004e7f000, 0xbba, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0041c5500, 0x29d, 0x29d)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0022cf3c0, 0xaa04df, 0x2, 0xc00136e400, 0xbba, 0x1, 0x29d, 0xc005bcea80)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00136e400, 0xbba, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0022cf518, 0xc001396f60, 0x38, 0xc004396cc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377680, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1167 [semacquire]:
github.com/advancedlogic/GoOse.Parser.setAttr(0xc00135bf80, 0xab29a3, 0xc, 0xacdde2, 0x2)
/home/develop/go/src/github.com/advancedlogic/GoOse/parser.go:87 +0x301
github.com/advancedlogic/GoOse.(*ContentExtractor).updateScore(0xc0020dd518, 0xc00135bf80, 0x29)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:493 +0xb1
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0020dd518, 0xc0059f4f60, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:435 +0x67f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000377d40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1168 [semacquire]:
strings.genSplit(0xc0035b1000, 0xe2e, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc006590000, 0x2af, 0x2af)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0037593c0, 0xaa0503, 0x2, 0xc005b2a000, 0xe2e, 0x3, 0x2af, 0xc0024886a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005b2a000, 0xe2e, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003759518, 0xc0034d1f20, 0x4d, 0xc00226bcc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004264e0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1170 [semacquire]:
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc001307170, 0xc001307138, 0xc0013071c0, 0x42ca96, 0xc00323d260, 0x30)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:824 +0x69
github.com/andybalholm/cascadia.ParseGroup(0xaac03e, 0x7, 0x1, 0xc0013071d8, 0xc0013071f0, 0x42d308, 0x30)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xaac03e, 0x7, 0x1, 0xb856e0, 0xc00608d720)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc00596eb70, 0xaac03e, 0x7, 0xc00323d260)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs(0xc0013076b0, 0xc0059647a0, 0xaac03e, 0x7, 0xc0059647a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:493 +0x81
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc0013076b0, 0xc0059647a0, 0xc00522db20)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:302 +0x39d
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000426f60, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1171 [semacquire]:
bytes.makeSlice(0x175a, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc001098ba0, 0x98, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc001098ba0, 0xc0031372c0, 0x98, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc001098ba0, 0xc0031372c0, 0x98, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc001098ba0, 0xc0046dac40, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc001098ba0, 0xc0046dabd0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001098ba0, 0xc000b18bd0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001098ba0, 0xc000b18af0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001098ba0, 0xc000b183f0, 0xb84d20, 0xc001098ba0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc001098ba0, 0xc000b183f0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc004d5df80, 0x19468c66f, 0x1251a20, 0xc00229d4e0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427260, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1172 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc004d5d230, 0x1942a0f97, 0x1251a20, 0xc00229ca00, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427560, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1173 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc00151d790, 0xc00664cfc0, 0x3e)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc00151d790, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc00151d790)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc00151d790, 0xc0052da380, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0000b3fe0, 0x0, 0x0, 0x0, 0xc000499200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0000b3fe0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0004998f8, 0xc000f1f800, 0x7b7, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000427e60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1174 [semacquire]:
strings.genSplit(0xc00148bb00, 0x858, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0054ca000, 0x1b4, 0x1b4)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000b933c0, 0xaa0599, 0x2, 0xc001d8cd00, 0x858, 0x1, 0x1b4, 0xc00229d200)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001d8cd00, 0x858, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000b93518, 0xc002226c60, 0x3e, 0xc000ccd080)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087c060, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1175 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00362d3c0, 0xaa04f9, 0x2, 0xc005aea000, 0x9f3, 0x5, 0x1c6, 0xc005a654a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005aea000, 0x9f3, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00362d518, 0xc0059658c0, 0x40, 0xc0021769cb)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087c120, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1176 [runnable]:
github.com/PuerkitoBio/goquery.newDocument(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:92
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc005048660, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:57 +0x77
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00375d8f8, 0xc002dab900, 0x12a2, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087c1e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1177 [semacquire]:
github.com/PuerkitoBio/goquery.appendWithoutDuplicates(0x0, 0x0, 0x0, 0xc001d080c0, 0x6, 0x6, 0xc000ac6fe0, 0xc005a65c80, 0x20, 0x20)
/home/develop/go/src/github.com/PuerkitoBio/goquery/utilities.go:137 +0x1cf
github.com/PuerkitoBio/goquery.mapNodes(0xc003255528, 0x1, 0x1, 0xc000ac70b0, 0x1, 0x0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:694 +0x159
github.com/PuerkitoBio/goquery.findWithMatcher(0xc003255528, 0x1, 0x1, 0xb856e0, 0xc005a65c80, 0xc00291df50, 0xc00291df50, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:543 +0x81
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc003738120, 0xaa408e, 0x4, 0xaa04f9)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x85
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:63 +0xd5
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc000ac78f8, 0xc00081a000, 0xd8f, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087c2a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1178 [semacquire]:
strings.Replace(0xc0023cd000, 0x2e4d, 0xaa5837, 0x5, 0xaa8176, 0x6, 0x5, 0xc0023cd000, 0x2e4d)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:177 +0x108
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0023998f8, 0xc003eaf000, 0x2e4b, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087c360, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1179 [semacquire]:
bytes.makeSlice(0xe05, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc001914b40, 0xfb, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc001914b40, 0xc001a59500, 0xfb, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc001914b40, 0xc001a59500, 0xfb, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc001914b40, 0xc003cec620, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc001914b40, 0xc003cec5b0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001914b40, 0xc00569ee00, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001914b40, 0xc00569ed20, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001914b40, 0xc00569e620, 0xb84d20, 0xc001914b40)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc001914b40, 0xc00569e620, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc001914990, 0x1949c8cae, 0x1251a20, 0xc00608d020, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087c420, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1180 [semacquire]:
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:109 +0x42
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087c4e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1181 [semacquire]:
golang.org/x/net/html/atom.String(0xc0065e80ff, 0x8, 0xf01, 0xf01, 0xc0065e8109)
/home/develop/go/src/golang.org/x/net/html/atom/atom.go:77 +0xc6
golang.org/x/net/html.(*Tokenizer).Token(0xc00092add0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x12a
golang.org/x/net/html.(*parser).parse(0xc00092aea0, 0xc000ad9c70, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc001000000, 0x0, 0x0, 0x0, 0xc004597200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc001000000, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0045978f8, 0xc002d22000, 0x971, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087c5a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1182 [semacquire]:
golang.org/x/net/html.NewTokenizerFragment(0xb7c280, 0xc00656cfe0, 0x0, 0x0, 0x20)
/home/develop/go/src/golang.org/x/net/html/token.go:1215 +0x43
golang.org/x/net/html.NewTokenizer(...)
/home/develop/go/src/golang.org/x/net/html/token.go:1201
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00656cfe0, 0x0, 0x0, 0x0, 0xc004f95200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2337 +0x41
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00656cfe0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc004f958f8, 0xc00086c380, 0xd43, 0x0, 0x0, 0xc00005b4f8)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087c660, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1183 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc004ab7000, 0xed3, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:71 +0x129
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc004ab7000, 0xed3, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc004ab7000, 0xed3, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc004ab7000, 0xed3, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc004ab7000, 0xed3, 0x0, 0x0, 0xed3, 0xc004ab7000, 0xed3)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc004ab7000, 0xed3, 0xed3, 0x1300)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00535c080, 0x3d, 0xc004ab7000, 0xed3, 0xc004ab7000, 0xed3, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00535c080, 0x3d, 0xc004ab7000, 0xed3, 0xaa0378, 0x1, 0xc004ab7000, 0xed3, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087c720, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1184 [semacquire]:
strings.makeCutsetFunc(0xaa4e4b, 0x5, 0xa7c820)
/usr/local/go/src/strings/strings.go:830 +0x4a
strings.TrimLeft(0xc003b5ea68, 0x5, 0xaa4e4b, 0x5, 0x5, 0x0)
/usr/local/go/src/strings/strings.go:855 +0x61
golang.org/x/net/html.inHeadIM(0xc00256fba0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:626 +0x969
golang.org/x/net/html.(*parser).parseCurrentToken(0xc00256fba0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc00256fba0, 0xc004ad7c70, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc001000fe0, 0x0, 0x0, 0x0, 0xc004f93200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc001000fe0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc004f938f8, 0xc001de1900, 0xc30, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087c7e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1186 [semacquire]:
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0031c4ff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x67e
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0031c4ff8, 0x1, 0xc00497dbc0, 0xc0103e88700, 0x280460fec108)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0031c4ff8, 0x4, 0x243, 0xc002c72d01, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc002c72000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x42c59c, 0x7fe2380c7008, 0x1ff)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc001d08510, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0031c58f8, 0xc002a11000, 0xdb1, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087c960, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1187 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc00086c380, 0xce6, 0xc003755218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc00086c380, 0xce6, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0037553c0, 0xaa058b, 0x2, 0xc00086c380, 0xce6, 0x5, 0x247, 0xc00656d080)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00086c380, 0xce6, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003755518, 0xc0034d1f60, 0x23, 0xc000cd5e30)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087ca20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1188 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc004c39a60, 0xc0037412c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0037413c0, 0xaa0503, 0x2, 0xc0003cbb00, 0x852, 0x4, 0x1a1, 0xc001c5e980)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0003cbb00, 0x852, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003741518, 0xc001c5e5c0, 0x3c, 0xc000aa86c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087cae0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1189 [semacquire]:
strings.genSplit(0xc005f42000, 0xf52, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x1000, 0x0, 0xc003757360)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0037573c0, 0xaa0569, 0x2, 0xc005f3b000, 0xf52, 0xc005f3b000, 0xc0037573b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005f3b000, 0xf52, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003757518, 0xc005965380, 0x1a, 0xc00138a940)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087cba0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1190 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc001de9000, 0xb45, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:87 +0x379
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc001de9000, 0xb45, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc001de9000, 0xb45, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc001de9000, 0xb45, 0x0, 0x0, 0x115f500, 0x46e6ce, 0xc001de9000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc001de9000, 0xb45, 0x0, 0x0, 0xb45, 0xc00, 0xc001de9000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc001de9000, 0xb45, 0xb45, 0xc00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc004886d50, 0x2b, 0xc001de9000, 0xb45, 0xc001de9000, 0xb45, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc004886d50, 0x2b, 0xc001de9000, 0xb45, 0xaa0378, 0x1, 0xc001de9000, 0xb45, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087ccc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1191 [runnable]:
bytes.(*Buffer).WriteString(0xc002c37020, 0xc002f6ed00, 0xf1, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:180 +0x10f
golang.org/x/net/html.escape(0xb84d20, 0xc002c37020, 0xc002f6ed00, 0xf1, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc002c37020, 0xc000a038f0, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc002c37020, 0xc000a03880, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc002c37020, 0xc000a02fc0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc002c37020, 0xc000a02ee0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc002c37020, 0xc000a027e0, 0xb84d20, 0xc002c37020)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc002c37020, 0xc000a027e0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc002c36e10, 0x193015b96, 0x1251a20, 0xc001396340, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087d2c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1192 [runnable]:
bytes.makeSlice.func1()
/usr/local/go/src/bytes/buffer.go:224 +0x68
bytes.makeSlice(0x3e5, 0xc002602800, 0x3e5, 0x3e5)
/usr/local/go/src/bytes/buffer.go:229 +0x96
bytes.(*Buffer).grow(0xc003753388, 0x13d, 0x7)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc003753388, 0xc000ab3cc0, 0x13d, 0x5165d7, 0xc0027ba40b, 0xc0035f8870)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0009ceb60)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0009ceaf0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0009ce070)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0009cbf80)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0009cb880)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc002c377a0, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003753518, 0xc001396600, 0x19, 0xc002389440)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087d440, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1193 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc0029ed2b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc0029ed380, 0xc000937960, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc001c5e340, 0x0, 0x0, 0x0, 0xc005487200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc001c5e340, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0054878f8, 0xc0007c5200, 0xc1b, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087d5c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1194 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc003bd1040, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:901 +0x820
golang.org/x/net/html.(*parser).parseCurrentToken(0xc003bd1040)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc003bd1040, 0xc000a05d50, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc005048c00, 0x0, 0x0, 0x0, 0xc0043e1200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc005048c00, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0043e18f8, 0xc002e8e000, 0x13d5, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087d740, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1195 [semacquire]:
strings.genSplit(0xc006488000, 0x944, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0060e4000, 0x1d3, 0x1d3)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000e233c0, 0xaa058b, 0x2, 0xc003dc6000, 0x944, 0x5, 0x1d3, 0xc001c5e580)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003dc6000, 0x944, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000e23518, 0xc005b74060, 0x18, 0xc0059fee20)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087d8c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1196 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00522d340, 0xc000e0b2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000e0b3c0, 0xaa04f9, 0x2, 0xc00432a000, 0x4eb, 0x0, 0xc000cb6a80, 0x7fe260fee9b8)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00432a000, 0x4eb, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000e0b518, 0xc001dd2820, 0x24, 0xc0008a1b90)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087da40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1197 [runnable]:
regexp.(*Regexp).doExecute(0xc000149400, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc004886b10, 0x2f, 0x0, 0x2, ...)
/usr/local/go/src/regexp/exec.go:521 +0x59b
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc004886b10, 0x2f, 0x2, 0xc0069f41d0, 0x0, 0xc0069f4270, ...)
/usr/local/go/src/regexp/regexp.go:599 +0x1a5
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc004886b10, 0x2f, 0xaa0378, 0x1, 0xc005ab7e00, 0x4)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0069f43d8, 0xc00069a310, 0xc001aa04d0, 0xc0069f4308)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069f43d8, 0xc00069a2a0, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0069f50c0, 0xc00069a2a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:155 +0x7cc
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0069f50c0, 0xc00069a2a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069f50c0, 0xc00069a1c0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002f770c0, 0xc00069a1c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002f770c0, 0xc00069a1c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069f50c0, 0xc00069a0e0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002f770c0, 0xc00069a0e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002f770c0, 0xc00069a0e0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069f50c0, 0xc002ab33b0, 0xc003be6300, 0x7fe260fedc00)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002f770c0, 0xc002ab33b0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002f770c0, 0xc002ab33b0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0069f50c0, 0xc002ab32d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002f770c0, 0xc002ab32d0, 0x54b170, 0xc002ffa4e0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc002ab32d0, 0x0, 0x0, 0x0, 0x0, 0xc002ab32d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc004b1fd70, 0x0, 0x0, 0x0, 0x1300, 0xc005778c00, 0x11a2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc005778c00, 0x11a2, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc0035f85d0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002f77518, 0xc0031124e0, 0xc003112e00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087dbc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1198 [runnable]:
github.com/fatih/set.(*Set).Add(0xc001396320, 0xc00362b2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00362b3c0, 0xaa04df, 0x2, 0xc00338ad80, 0xd31, 0x1, 0x28e, 0xc001396300)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00338ad80, 0xd31, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00362b518, 0xc005c55de0, 0x1f, 0xc002388f40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087dd40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1199 [semacquire]:
strings.genSplit(0xc00648a000, 0x1448, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc005ef2000, 0x3ba, 0x3ba)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00063b3c0, 0xaa0503, 0x2, 0xc001a96a00, 0x1448, 0x4, 0x3ba, 0xc004cdba80)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001a96a00, 0x1448, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00063b518, 0xc005b742e0, 0x39, 0xc002403f00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00087dec0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1200 [semacquire]:
strings.genSplit(0xc0037ff000, 0x7cd, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x0, 0xc005b6ea80, 0xc001789360)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0017893c0, 0xaa04d9, 0x2, 0xc0037fe800, 0x7cd, 0xc0037fe800, 0xc0017893b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0037fe800, 0x7cd, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001789518, 0xc0059f5de0, 0x3c, 0xc002403848)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94000, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1201 [semacquire]:
strings.genSplit(0xc001aeb200, 0x585, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0016e2a00, 0x143, 0x143)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00375b3c0, 0xaa0503, 0x2, 0xc00465a000, 0x585, 0x2, 0x143, 0xc004c39f00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00465a000, 0x585, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00375b518, 0xc0013975a0, 0x3b, 0xc001bfb748)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94180, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1202 [semacquire]:
regexp.(*Regexp).expand(0xc000149400, 0xc000e71f00, 0x80, 0x80, 0xaa0378, 0x1, 0x0, 0x0, 0x0, 0xc005d02fd0, ...)
/usr/local/go/src/regexp/regexp.go:948 +0xa9b
regexp.(*Regexp).ReplaceAllString.func1(0xc000e71f00, 0x80, 0x80, 0xc0043f1e20, 0x2, 0x2, 0xc005d02fd0, 0xaf, 0x7d)
/usr/local/go/src/regexp/regexp.go:558 +0xbf
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc005d02fd0, 0xaf, 0x2, 0xc0021bc198, 0xc00000000a, 0x2, ...)
/usr/local/go/src/regexp/regexp.go:616 +0x2e9
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc005d02fd0, 0xaf, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021bd0c0, 0xc00242d110, 0xc0021bc1b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0021bd0c0, 0xc00242d0a0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0021bd0c0, 0xc00242d0a0, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0021bd0c0, 0xc00242d0a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021bd0c0, 0xc00242d0a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0021bd0c0, 0xc00242c930, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0021bd0c0, 0xc00242c930, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021bd0c0, 0xc00242c930, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0021bd0c0, 0xc00242c850, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0021bd0c0, 0xc00242c850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021bd0c0, 0xc00242c850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0021bd0c0, 0xc00242c150, 0xc0021bcd40, 0x46a81e)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0021bd0c0, 0xc00242c150, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021bd0c0, 0xc00242c150, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0021bd0c0, 0xc000bfbf80, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021bd0c0, 0xc000bfbf80, 0x54b170, 0xc000198c30)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc000bfbf80, 0x0, 0x0, 0x0, 0x0, 0xc000bfbf80, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc000593050, 0x0, 0x0, 0x0, 0xa80, 0xc0042f4a80, 0x93d, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0042f4a80, 0x93d, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc0049c3ce0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0021bd518, 0xc00039f720, 0xc001cfc0e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1203 [semacquire]:
github.com/PuerkitoBio/goquery.appendWithoutDuplicates(0xc00608c000, 0x4, 0x4, 0xc001d08390, 0x6, 0x6, 0xc00381b260, 0xc005a65fc0, 0x20, 0x20)
/home/develop/go/src/github.com/PuerkitoBio/goquery/utilities.go:137 +0x1cf
github.com/PuerkitoBio/goquery.mapNodes(0xc0035cdee8, 0x1, 0x1, 0xc00381b330, 0x1, 0x0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:694 +0x159
github.com/PuerkitoBio/goquery.findWithMatcher(0xc0035cdee8, 0x1, 0x1, 0xb856e0, 0xc005a65fc0, 0xc00381b3b0, 0x753f20, 0xaa3fc2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:543 +0x81
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0020500f0, 0xaa408e, 0x4, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x85
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00381b518, 0xc001396a60, 0x39, 0xc002dde340)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:126 +0x200
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1204 [semacquire]:
bytes.makeSlice(0xbe1, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc002f07ad0, 0x1, 0x2)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc002f07ad0, 0xad0190, 0x1, 0x2, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.render1(0xb84d20, 0xc002f07ad0, 0xc001735f80, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:228 +0x7b3
golang.org/x/net/html.render1(0xb84d20, 0xc002f07ad0, 0xc001735810, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc002f07ad0, 0xc001735730, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc002f07ad0, 0xc001735030, 0xb84d20, 0xc002f07ad0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc002f07ad0, 0xc001735030, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc001153c20, 0x20, 0x203000, 0x203000, 0x203000)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003793518, 0xc000c443a0, 0xc005bcf660)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:285 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f949c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1205 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc000c45540, 0xc00459f2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00459f3c0, 0xaa058b, 0x2, 0xc002b75800, 0x74e, 0x5, 0x17a, 0xc000c45520)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002b75800, 0x74e, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00459f518, 0xc000c45040, 0x39, 0xc0056875c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94b40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1206 [semacquire]:
github.com/PuerkitoBio/goquery.pushStack(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/utilities.go:159
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc000593710, 0xab9e11, 0x19, 0xc003060120)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0xb3
github.com/advancedlogic/GoOse.(*ContentExtractor).getTitleUnmodified(0xc00459b518, 0xc001cfc580, 0xe7d, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:63 +0x12b
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:128 +0x282
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f94fc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1207 [semacquire]:
regexp.(*Regexp).get(0xc000149cc0, 0xc000011601)
/usr/local/go/src/regexp/regexp.go:256 +0x322
regexp.(*Regexp).doExecute(0xc000149cc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0013becc0, 0x9, 0x0, 0x0, ...)
/usr/local/go/src/regexp/exec.go:538 +0xb5
regexp.(*Regexp).doMatch(...)
/usr/local/go/src/regexp/exec.go:514
regexp.(*Regexp).MatchString(...)
/usr/local/go/src/regexp/regexp.go:507
github.com/advancedlogic/GoOse.(*Cleaner).cleanBadTags.func1.1(0x0, 0xc0019158f0)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:427 +0xd0
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc0019158c0, 0xc000959250, 0x7)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).cleanBadTags.func1(0x1, 0xc001915830)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:425 +0x17d
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc001915740, 0xc0009592f8, 0x4)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).cleanBadTags(0xc0009596b0, 0xc005c55020, 0xc000149cc0, 0xc0009593a0, 0xc001e3de30)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:421 +0xa4
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc0009596b0, 0xc005c55020, 0xc005c55920)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:293 +0x199
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95200, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1208 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc004d80000, 0xae5, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc004d80000, 0xae5, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc004d80000, 0xae5, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc004d80000, 0xae5, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc004d80000, 0xae5, 0x0, 0x0, 0xae5, 0xc004d80000, 0xae5)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc004d80000, 0xae5, 0xae5, 0xc00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002402540, 0x38, 0xc004d80000, 0xae5, 0xc004d80000, 0xae5, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002402540, 0x38, 0xc004d80000, 0xae5, 0xaa0378, 0x1, 0xc004d80000, 0xae5, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95380, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1209 [semacquire]:
bytes.makeSlice(0xe00, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0027490e8, 0x200, 0x400)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).ReadFrom(0xc0027490e8, 0xb7b620, 0xc001a6e180, 0x203000, 0x203000, 0x203000)
/usr/local/go/src/bytes/buffer.go:202 +0x48
io/ioutil.readAll(0xb7b620, 0xc001a6e180, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:36 +0xe3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc001a6e180, 0xc0027491d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc001a6e180, 0x0, 0x0, 0x0, 0x900, 0xc000158900, 0x83d, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc000158900, 0x83d, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc002402a80)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002749518, 0xc005b74b40, 0xc005b753a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95500, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1210 [semacquire]:
strings.Replace(0xc005b12000, 0x3fe0, 0xaa042f, 0x2, 0xaa0af0, 0x3, 0x2, 0xc03f800c60, 0x0)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:175 +0x72
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc002bfd8f8, 0xc005b12000, 0x3fe0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95680, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1211 [semacquire]:
strings.Replace(0xc003923000, 0x166d, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0x13, 0xc003923000, 0x166d)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0021d78f8, 0xc0006a5000, 0x166b, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95800, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1212 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc0007bf500, 0x1477, 0x5165d7, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc0007bf500, 0x1477, 0xc00239d1f8, 0x4)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0037973c0, 0xaa04f9, 0x2, 0xc0007bf500, 0x1477, 0xc0007bf500, 0xc0037973b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0007bf500, 0x1477, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003797518, 0xc003213220, 0x3b, 0xc00345694e)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95980, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1213 [runnable]:
bytes.makeSlice.func1()
/usr/local/go/src/bytes/buffer.go:224 +0x68
bytes.makeSlice(0x5a7, 0xc002d5e000, 0x5a7, 0x5a7)
/usr/local/go/src/bytes/buffer.go:229 +0x96
bytes.(*Buffer).grow(0xc0012ed208, 0x107, 0xf0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0012ed208, 0xc0053eb200, 0x107, 0x5165d7, 0xc000975140, 0xc005fc3e98)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc005fcd2d0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc005fcd260)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc005fcc9a0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc005c45b60, 0xc005c45b60, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0012ed518, 0xc004c39b80, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:423 +0x38b
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95b00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1214 [semacquire]:
golang.org/x/net/html.(*Tokenizer).readTag(0xc000899860, 0xc002fd8701)
/home/develop/go/src/golang.org/x/net/html/token.go:834 +0x162
golang.org/x/net/html.(*Tokenizer).readStartTag(0xc000899860, 0x7fe20000006d)
/home/develop/go/src/golang.org/x/net/html/token.go:779 +0x45
golang.org/x/net/html.(*Tokenizer).Next(0xc000899860, 0x1)
/home/develop/go/src/golang.org/x/net/html/token.go:1021 +0x28c
golang.org/x/net/html.(*parser).parse(0xc000899930, 0xc0013ade30, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2284 +0xad
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc000c45a40, 0x0, 0x0, 0x0, 0xc002745200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc000c45a40, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0027458f8, 0xc00285e800, 0x710, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95c80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1215 [semacquire]:
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc001098f60, 0x1947bacc9, 0x1251a20, 0xc00608c280, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:100 +0x31
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95e00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1216 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc002873380, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:908 +0x1761
golang.org/x/net/html.(*parser).parseCurrentToken(0xc002873380)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc002873380, 0xc003ced650, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc001884be0, 0x0, 0x0, 0x0, 0xc0045a3200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc001884be0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0045a38f8, 0xc002f5e000, 0xfb9, 0xc001432724, 0x3, 0xc001432728)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc001f95f80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1217 [semacquire]:
bytes.NewReader(...)
/usr/local/go/src/bytes/reader.go:160
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc0041ee1b0, 0xc0020431d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:33 +0xc3
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0041ee1b0, 0x0, 0x0, 0x0, 0xd80, 0xc004c7c880, 0xcee, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc004c7c880, 0xcee, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc004397700)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002043518, 0xc0032a94a0, 0xc0032a9e40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002256240, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1218 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.afterHeadIM(0xc000199ba0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:780 +0x232
golang.org/x/net/html.(*parser).parseCurrentToken(0xc000199ba0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc000199ba0, 0xc0064bfb90, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc003383440, 0x0, 0x0, 0x0, 0xc0027cb200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc003383440, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0027cb8f8, 0xc002966600, 0x11e0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002256300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1219 [runnable]:
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc002f701c0, 0xc00654e7e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:127 +0x159
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc002f701c0, 0xc00654e770, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:133 +0x9e
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc002f701c0, 0xc00654e2a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:133 +0x9e
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc002f701c0, 0xc00654e1c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:133 +0x9e
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc002f701c0, 0xc003d5a150, 0x0, 0x0, 0x0, 0x10, 0x9ed160, 0x46a5ec)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:133 +0x9e
github.com/andybalholm/cascadia.Selector.MatchAll(0xc002f701c0, 0xc003d5a150, 0xc0027cd1e8, 0x747325, 0xc000151c00)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:124 +0x46
github.com/PuerkitoBio/goquery.findWithMatcher.func1(0x0, 0xc003d5a070, 0x7fe2287efcf1, 0x40, 0xe)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:547 +0x8e
github.com/PuerkitoBio/goquery.mapNodes(0xc000889a88, 0x1, 0x1, 0xc0027cd348, 0x1, 0x0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:693 +0xf6
github.com/PuerkitoBio/goquery.findWithMatcher(0xc000889a88, 0x1, 0x1, 0xb856e0, 0xc002f701c0, 0xc0027cd3c8, 0x46e6ce, 0xc005eb4880)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:543 +0x81
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0065361b0, 0xaa64ee, 0x5, 0x18a2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x85
github.com/advancedlogic/GoOse.(*ContentExtractor).getTitleUnmodified(0xc0027cd518, 0xc001884fc0, 0xc85, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:57 +0x47
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:128 +0x282
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0022563c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1220 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc0028731e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x18a
golang.org/x/net/html.(*parser).parse(0xc0028732b0, 0xc00057ca10, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7b620, 0xc0005acff0, 0x0, 0x0, 0x0, 0x0, 0xd01, 0xc0005acfc0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0005acfc0, 0x0, 0x0, 0x0, 0x1000, 0xc003b28000, 0xdf5, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:93 +0x76
github.com/jaytaylor/html2text.FromString(0xc003b28000, 0xdf5, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc002177a40)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc004477518, 0xc00229ca20, 0xc004cda120)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002256480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1221 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc00092b2b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc00092b380, 0xc002ea8690, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc005d7b100, 0x0, 0x0, 0x0, 0xc00541b200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc005d7b100, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00541b8f8, 0xc001c77000, 0x36fe, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002256540, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1222 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005ca6740, 0xc0054852c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0054853c0, 0xaa04fd, 0x2, 0xc0063ce400, 0xb1f, 0x1, 0x22b, 0xc00103f3a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0063ce400, 0xb1f, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc005485518, 0xc00103eec0, 0x3c, 0xc000efe7c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002256660, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1223 [semacquire]:
regexp.(*Regexp).get(0xc000149cc0, 0xc000622401)
/usr/local/go/src/regexp/regexp.go:256 +0x322
regexp.(*Regexp).doExecute(0xc000149cc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0001e8d90, 0x9, 0x0, 0x0, ...)
/usr/local/go/src/regexp/exec.go:538 +0xb5
regexp.(*Regexp).doMatch(...)
/usr/local/go/src/regexp/exec.go:514
regexp.(*Regexp).MatchString(...)
/usr/local/go/src/regexp/regexp.go:507
github.com/advancedlogic/GoOse.(*Cleaner).cleanBadTags.func1.1(0x0, 0xc005cf4210)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:427 +0xd0
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc005cf41e0, 0xc001e6b250, 0x7)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).cleanBadTags.func1(0x1, 0xc005cf4090)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:425 +0x17d
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc005b3fe90, 0xc001e6b2f8, 0x4)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).cleanBadTags(0xc001e6b6b0, 0xc001cfd3c0, 0xc000149cc0, 0xc001e6b3a0, 0xc005764e70)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:421 +0xa4
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc001e6b6b0, 0xc001cfd3c0, 0xc001c5ef40)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:293 +0x199
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002256720, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1224 [semacquire]:
github.com/PuerkitoBio/goquery.pushStack(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/utilities.go:159
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc005c91b90, 0xaa3ff6, 0x4, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0xb3
github.com/advancedlogic/GoOse.(*ContentExtractor).GetFavicon(0xc004df7518, 0xc005ca6460, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:161 +0x4f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:131 +0x34d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0022567e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1225 [semacquire]:
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc005c54000, 0xc0064bc3f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:129 +0x11e
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc005c54000, 0xc0064bc310, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:133 +0x9e
github.com/andybalholm/cascadia.Selector.matchAllInto(0xc005c54000, 0xc006065c00, 0x0, 0x0, 0x0, 0xc001faad80, 0x5, 0xc003cc2ff8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:133 +0x9e
github.com/andybalholm/cascadia.Selector.MatchAll(0xc005c54000, 0xc006065c00, 0x466665, 0xc002314a80, 0x5)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:124 +0x46
github.com/PuerkitoBio/goquery.findWithMatcher.func1(0x0, 0xc006065b20, 0x0, 0x8, 0x7fe23806b560)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:547 +0x8e
github.com/PuerkitoBio/goquery.mapNodes(0xc000513188, 0x1, 0x1, 0xc003cc31d8, 0x1, 0x0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:693 +0xf6
github.com/PuerkitoBio/goquery.findWithMatcher(0xc000513188, 0x1, 0x1, 0xb856e0, 0xc005c54000, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:543 +0x81
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc00649a7e0, 0xaac03e, 0x7, 0xc0028c6f90)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x85
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs(0xc003cc36b0, 0xc003382ca0, 0xaac03e, 0x7, 0xc003382ca0)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:493 +0x81
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc003cc36b0, 0xc003382ca0, 0xc005834c80)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:302 +0x39d
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0022568a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1226 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc000387040, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:901 +0x820
golang.org/x/net/html.(*parser).parseCurrentToken(0xc000387040)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc000387040, 0xc0022a0f50, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc001397a80, 0x0, 0x0, 0x0, 0xc003b49200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc001397a80, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc003b498f8, 0xc0029c9000, 0x7e8, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002256b40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1227 [semacquire]:
regexp/syntax.(*compiler).inst(...)
/usr/local/go/src/regexp/syntax/compile.go:173
regexp/syntax.(*compiler).rune(0xc003b4ab30, 0xc00112eb48, 0x1, 0xe, 0xd5, 0x600000003)
/usr/local/go/src/regexp/syntax/compile.go:267 +0x27c
regexp/syntax.(*compiler).compile(0xc003b4ab30, 0xc005861180, 0x200000001)
/usr/local/go/src/regexp/syntax/compile.go:110 +0x12c
regexp/syntax.(*compiler).compile(0xc003b4ab30, 0xc0058611f0, 0x0)
/usr/local/go/src/regexp/syntax/compile.go:156 +0xb29
regexp/syntax.Compile(0xc0058611f0, 0xc0058611f0, 0x1, 0x1)
/usr/local/go/src/regexp/syntax/compile.go:83 +0x10a
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:178 +0xd7
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc003b4aff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc003b4aff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc0035f8138)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc003b4aff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc003b4aff8, 0x1, 0xc0013ae450, 0xc0104303180, 0x280460fece98)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc003b4aff8, 0x4, 0x526, 0xc00081ba01, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc00081b000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc00266ec30, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc002920a80, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc003b4b8f8, 0xc000a91400, 0xad4, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002256c00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1228 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00103ee20, 0xc003baf2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003baf3c0, 0xaa0503, 0x2, 0xc00160fc00, 0x38d, 0x2, 0xee, 0xc00103ee00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00160fc00, 0x38d, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003baf518, 0xc001c5ff60, 0x3b, 0xc001732840)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002256cc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1229 [semacquire]:
bytes.makeSlice(0x40f, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00174d0c0, 0x1, 0x1)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00174d0c0, 0x1189d45, 0x1, 0x1, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc00174d0c0, 0xc004add790, 0xca, 0xca, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:440 +0xf7
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00174d0c0, 0xc001043960, 0xc00174c1b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00174d0c0, 0xc0010438f0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc00174d0c0, 0xc0010438f0, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00174d0c0, 0xc0010438f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00174d0c0, 0xc0010438f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00174d0c0, 0xc001043180, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002b9d0c0, 0xc001043180, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002b9d0c0, 0xc001043180, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00174d0c0, 0xc0010430a0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002b9d0c0, 0xc0010430a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002b9d0c0, 0xc0010430a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00174d0c0, 0xc0010429a0, 0xc001a0dbc8, 0x4)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002b9d0c0, 0xc0010429a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002b9d0c0, 0xc0010429a0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00174d0c0, 0xc0010428c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002b9d0c0, 0xc0010428c0, 0x54b170, 0xc00092ad00)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc0010428c0, 0x0, 0x0, 0x0, 0x0, 0xc0010428c0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc000aa6960, 0x0, 0x0, 0x0, 0x800, 0xc005e2b800, 0x79c, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc005e2b800, 0x79c, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xdf2f3a700000000)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002b9d518, 0xc005b758c0, 0xc005d7a220)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002256d80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1230 [semacquire]:
regexp/syntax.(*parser).push(0xc0047d8d80, 0xc005e7a000, 0xc005e7a000)
/usr/local/go/src/regexp/syntax/parse.go:139 +0x1a2
regexp/syntax.(*parser).literal(0xc0047d8d80, 0x6f)
/usr/local/go/src/regexp/syntax/parse.go:210 +0x53
regexp/syntax.Parse(0xac0e09, 0x13, 0xd4, 0x0, 0x0, 0xac0dfd)
/usr/local/go/src/regexp/syntax/parse.go:729 +0x2ae
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:170 +0x5a
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0027ceff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0027ceff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc005e5e848)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0027ceff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0027ceff8, 0x1, 0xc002b34d00, 0xc0101d7aa00, 0x280460fee9b8)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0027ceff8, 0x4, 0xc94, 0xc00211d301, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc00211d000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc00151d110, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0029a6120, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0027cf8f8, 0xc0020e6a00, 0x1355, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002256e40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1231 [runnable]:
regexp/syntax.(*Inst).MatchRunePos(0xc0002053e8, 0xc000000070, 0x20)
/usr/local/go/src/regexp/syntax/prog.go:201 +0x25e
regexp/syntax.(*Inst).MatchRune(...)
/usr/local/go/src/regexp/syntax/prog.go:193
regexp.(*machine).step(0xc002d30000, 0xc002d30010, 0xc002d30040, 0x0, 0x1, 0x70, 0xc00672ef58)
/usr/local/go/src/regexp/exec.go:295 +0x5d1
regexp.(*machine).match(0xc002d30000, 0xb886e0, 0xc002d300c0, 0x0, 0xc0068006f0)
/usr/local/go/src/regexp/exec.go:225 +0x21f
regexp.(*Regexp).doExecute(0xc000149cc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc006332520, 0x9, 0x0, 0x0, ...)
/usr/local/go/src/regexp/exec.go:542 +0x177
regexp.(*Regexp).doMatch(...)
/usr/local/go/src/regexp/exec.go:514
regexp.(*Regexp).MatchString(...)
/usr/local/go/src/regexp/regexp.go:507
github.com/advancedlogic/GoOse.(*Cleaner).cleanBadTags.func1.1(0x0, 0xc00549eb10)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:427 +0xd0
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc00549eae0, 0xc00672f250, 0x7)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).cleanBadTags.func1(0x1, 0xc00549ea50)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:425 +0x17d
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc00549e960, 0xc00672f2f8, 0x4)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).cleanBadTags(0xc00217f6b0, 0xc0063e6220, 0xc000149cc0, 0xc00217f3a0, 0xc004cb0d50)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:421 +0xa4
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc00672f6b0, 0xc0063e6220, 0xc00229d440)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:293 +0x199
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002256f00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1232 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc00452e9c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1186 +0x35f
golang.org/x/net/html.(*parser).parse(0xc00452ea90, 0xc0027b6310, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00522ca60, 0x0, 0x0, 0x0, 0xc00425b200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00522ca60, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00425b8f8, 0xc002d4a400, 0xb18, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002256fc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1233 [semacquire]:
strings.genSplit(0xc0049c7a80, 0x1987, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc00150e000, 0x4e3, 0x4e3)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0046133c0, 0xaa04df, 0x2, 0xc00443fa80, 0x1987, 0x6, 0x4e3, 0xc00000f060)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00443fa80, 0x1987, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004613518, 0xc00000e9c0, 0x3b, 0xc002e2ae40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002257080, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1234 [semacquire]:
bytes.makeSlice(0xcc, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc002a7f208, 0xcc, 0x203000)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc002a7f208, 0xc002f8fad0, 0xcc, 0x1, 0xb856e0, 0xc00608cb00)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0004d3c00)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0004d3b90)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc001a6e4e0, 0xa9ece0, 0xc001a6e4b0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc002a7f518, 0xc0034d0ce0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:384 +0x144
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002257140, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1235 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc004d62000, 0x1af7, 0xc00551b218, 0xc001d7aa80)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc004d62000, 0x1af7, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00551b3c0, 0xaa0599, 0x2, 0xc004d62000, 0x1af7, 0x1, 0x502, 0xc004908da0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004d62000, 0x1af7, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00551b518, 0xc005048060, 0x38, 0xc00322e700)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002257200, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1236 [semacquire]:
golang.org/x/net/html.(*parser).addChild(0xc00078c270, 0xc00250c000)
/home/develop/go/src/golang.org/x/net/html/parse.go:234 +0xf3
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc00078c270, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:1097 +0x76a
golang.org/x/net/html.(*parser).parseCurrentToken(0xc00078c270)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc00078c270, 0xc0003e08c0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7b620, 0xc003988a50, 0x0, 0x0, 0x0, 0x0, 0xf01, 0xc0034ad680)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0034ad680, 0x0, 0x0, 0x0, 0x1000, 0xc003bd2000, 0xfa9, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:93 +0x76
github.com/jaytaylor/html2text.FromString(0xc003bd2000, 0xfa9, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc004750040)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00252d518, 0xc000fef380, 0xc000fefd60)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0022572c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1237 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc0028720d0, 0xc00360a000, 0xa)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc0028720d0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0028720d0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0028720d0, 0xc0004248c0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0000b29a0, 0x0, 0x0, 0x0, 0xc003811200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0000b29a0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0038118f8, 0xc002658000, 0x15ec, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002257560, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1238 [semacquire]:
github.com/gigawattio/window.Rolling(0xc00625c000, 0xdd, 0xdd, 0x3, 0x0, 0xdc, 0xc00625c000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00449d518, 0xc005965320, 0xc002408600)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc002257620, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1239 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0024bf588, 0xaa04f9, 0x2, 0xc0030ab800, 0x163, 0x1, 0xc, 0xc005ca65c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc000f45518, 0xc0018856a0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4000, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1240 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc00437ea80, 0x906, 0xc000c03218, 0xc002c4a700)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc00437ea80, 0x906, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000c033c0, 0xaa0503, 0x2, 0xc00437ea80, 0x906, 0x4, 0x1bd, 0xc001cfc900)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00437ea80, 0x906, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000c03518, 0xc005964b20, 0x3b, 0xc000aa8400)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4180, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1241 [semacquire]:
github.com/PuerkitoBio/goquery.newSingleSelection(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:113
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc005a62cf0, 0xc0068a91d8, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x9b
github.com/advancedlogic/GoOse.(*ContentExtractor).nodesToCheck(0xc0068a9518, 0xc005048360, 0x28, 0xc005a62c90, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:551 +0x183
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0068a9518, 0xc005048360, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:374 +0x50
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1242 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc0029ec270, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc0029ec340, 0xc00078bb90, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0034d1e20, 0x0, 0x0, 0x0, 0xc001b6b200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0034d1e20, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc001b6b8f8, 0xc003664000, 0x1867, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1243 [semacquire]:
bytes.makeSlice(0x1127, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00191aa50, 0xe9, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00191aa50, 0xc001cf3590, 0xe9, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc00191aa50, 0xc001cf3590, 0xe9, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc00191aa50, 0xc0054981c0, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc00191aa50, 0xc005498150, 0xb84d20, 0xc00191aa50)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc00191aa50, 0xc005498150, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc00191aa20, 0x203000, 0x203000, 0xc31cb19e00000000, 0xc00681d138)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs.func1(0x0, 0xc00191aa20)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:496 +0x67
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc00191a9f0, 0xc00681d2c8, 0x7)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs(0xc00681d6b0, 0xc005048ae0, 0xaac03e, 0x7, 0xc005048ae0)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:495 +0xd8
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc00681d6b0, 0xc005048ae0, 0xc000c44a00)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:302 +0x39d
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f4600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1244 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/jaytaylor/html2text.FromHTMLNode(0xc00133d490, 0x0, 0x0, 0x0, 0x0, 0xc00133d490, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:81 +0x10d
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc000e51200, 0x0, 0x0, 0x0, 0xa80, 0xc001360a80, 0x93b, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc001360a80, 0x93b, 0x0, 0x0, 0x0, 0xffffffffffffffff, 0xad0464, 0x3f00006b00000004, 0xc0011276d0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0043cf518, 0xc005d7a7a0, 0xc005d7b020)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f5440, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1245 [runnable]:
strings.ToLower(0xc003786000, 0x850, 0x7fe23833ffd8, 0xc00275f2e0)
/usr/local/go/src/strings/strings.go:584 +0x2dc
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00275f3c0, 0xaa04df, 0x2, 0xc003786000, 0x850, 0xc003786000, 0xc00275f3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003786000, 0x850, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00275f518, 0xc0024085a0, 0x31, 0xc005686f80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f55c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1246 [runnable]:
golang.org/x/net/html.(*Tokenizer).readByte(0xc003bd1790, 0x68)
/home/develop/go/src/golang.org/x/net/html/token.go:234 +0x440
golang.org/x/net/html.(*Tokenizer).Next(0xc003bd1790, 0xc0100000002)
/home/develop/go/src/golang.org/x/net/html/token.go:980 +0xcd
golang.org/x/net/html.(*parser).parse(0xc003bd1860, 0xc001d03dc0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2284 +0xad
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0000b2fc0, 0x0, 0x0, 0x0, 0xc003193200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0000b2fc0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0031938f8, 0xc00399bc00, 0x63e, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f5b00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1247 [semacquire]:
regexp.makeOnePass(0xc002c37410, 0xc002c37410)
/usr/local/go/src/regexp/onepass.go:309 +0x20e
regexp.compileOnePass(0xc002c373b0, 0xc002c373b0)
/usr/local/go/src/regexp/onepass.go:500 +0x129
regexp.compile(0xabc10c, 0x11, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0020ed300, 0x0, 0xabc106, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0020ed300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0020ed300, 0x0, 0xb856e0, 0xc00039e048, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0020ed300, 0xc0036d36c0, 0x0, 0x8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0020ed300, 0x42ca96, 0xc002c37380, 0x30, 0x28, 0xa4b3a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xabc101, 0x1d, 0xc0020ed370, 0x42d308, 0x30, 0xa4b3a0, 0xb85601)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xabc101, 0x1d, 0xc0059f4380, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0004d0bd0, 0xabc101, 0x1d, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc0020ed518, 0xc004cda1e0, 0xabc101, 0x1d, 0x3b, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:133 +0x3a7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f5c80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1248 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc001074000, 0x477, 0xc00098f218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc001074000, 0x477, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00098f3c0, 0xaa05c9, 0x2, 0xc001074000, 0x477, 0x2, 0x113, 0xc005a65180)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001074000, 0x477, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00098f518, 0xc0031136a0, 0x37, 0xc0056871c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f5e00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1249 [semacquire]:
bytes.makeSlice(0x51d, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc002183388, 0x11b, 0x87)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc002183388, 0xc003d7b320, 0x11b, 0x5165d7, 0xc002c72cf2, 0xc0012ef2a8)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000b742a0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000b74230)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000b71960)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000b71880)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc000b71180)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc003b37d70, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002183518, 0xc002226780, 0x38, 0xc000ccc500)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0018f5f80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1250 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc0041fb800, 0x1653, 0xc001715218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc0041fb800, 0x1653, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0017153c0, 0xaa0599, 0x2, 0xc0041fb800, 0x1653, 0x1, 0x482, 0xc00137ed00)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0041fb800, 0x1653, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001715518, 0xc0059f5fc0, 0x3c, 0xc000117940)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061eae0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1251 [runnable]:
github.com/gigawattio/window.Rolling(0xc004ad9000, 0xe9, 0xe9, 0x3, 0x0, 0xe8, 0xc004ad9000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:20 +0xe8
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00070b518, 0xc002227620, 0xc002227f20)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061f740, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1252 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc000869b00, 0xcee, 0xc00444d218, 0xc0009e5180)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc000869b00, 0xcee, 0x1, 0x0)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00444d3c0, 0xaa04f9, 0x2, 0xc000869b00, 0xcee, 0x5, 0x28d, 0xc0059f4f40)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000869b00, 0xcee, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00444d518, 0xc0059f4c00, 0x4c, 0xc00226bc20)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061fa40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1253 [semacquire]:
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc00040a800, 0x1fa, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:560 +0x135
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0022d30c0, 0xc0015fab60, 0xc0022d1e48, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0022d30c0, 0xc0015faaf0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0022d30c0, 0xc0015faaf0, 0xc000000001, 0xa)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0022d30c0, 0xc0015faaf0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0022d30c0, 0xc0015faaf0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0022d30c0, 0xc0015fa770, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0022d30c0, 0xc0015fa770, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0022d30c0, 0xc0015fa770, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0022d30c0, 0xc001467030, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00070b0c0, 0xc001467030, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00070b0c0, 0xc001467030, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0022d30c0, 0xc001466f50, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00070b0c0, 0xc001466f50, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00070b0c0, 0xc001466f50, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0022d30c0, 0xc001466850, 0xc0033de380, 0x7fe260fece98)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00070b0c0, 0xc001466850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00070b0c0, 0xc001466850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0022d30c0, 0xc001466770, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00070b0c0, 0xc001466770, 0x54b170, 0xc003ed1ba0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc001466770, 0x0, 0x0, 0x0, 0x0, 0xc001466770, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc003989c20, 0x0, 0x0, 0x0, 0x1b00, 0xc003fac700, 0x1aeb, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc003fac700, 0x1aeb, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc004396480)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00070b518, 0xc001396020, 0xc001396e40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc00061fe00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1254 [semacquire]:
bytes.makeSlice(0x171, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00191a750, 0x1, 0xc000142130)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteByte(0xc00191a750, 0xc00002033e, 0xad0190, 0x1)
/usr/local/go/src/bytes/buffer.go:267 +0x8c
golang.org/x/net/html.render1(0xb84d20, 0xc00191a750, 0xc0053fd490, 0xb84d20, 0xc00191a750)
/home/develop/go/src/golang.org/x/net/html/render.go:183 +0x56c
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc00191a750, 0xc0053fd490, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc00191a6f0, 0x203000, 0x203000, 0x10349a8100000000, 0xc006935138)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs.func1(0x0, 0xc00191a6f0)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:496 +0x67
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc00191a6c0, 0xc0069352c8, 0x7)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs(0xc0069356b0, 0xc0050483a0, 0xaac03e, 0x7, 0xc0050483a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:495 +0xd8
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc0069356b0, 0xc0050483a0, 0xc00115b3a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:302 +0x39d
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8000, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1255 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.glob..func16(0xc000000065, 0xf00)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:364 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectScript(0xc0065e2000, 0xfc6, 0x2)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:271 +0x3a2
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0065e2000, 0xfc6, 0x0, 0x0, 0xfc6, 0xc0065e2000, 0xfc6)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:58 +0x39
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0065e2000, 0xfc6, 0xfc6, 0x1000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001bfab80, 0x34, 0xc0065e2000, 0xfc6, 0xc0065e2000, 0xfc6, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001bfab80, 0x34, 0xc0065e2000, 0xfc6, 0xaa0378, 0x1, 0xc0065e2000, 0xfc6, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8180, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1256 [semacquire]:
regexp/syntax.(*parser).push(0xc002c0ccc0, 0xc003ba7f80, 0xc0043d2d01)
/usr/local/go/src/regexp/syntax/parse.go:139 +0x1a2
regexp/syntax.(*parser).op(0xc002c0ccc0, 0xaba609, 0xe)
/usr/local/go/src/regexp/syntax/parse.go:218 +0x84
regexp/syntax.Parse(0xaba6cc, 0xe, 0xd4, 0x0, 0x0, 0xaba6c6)
/usr/local/go/src/regexp/syntax/parse.go:754 +0xd16
regexp.compile(0xaba6cc, 0xe, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:170 +0x5a
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0043d3300, 0x0, 0xaba6c6, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0043d3300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0043d3300, 0x0, 0x0, 0x0, 0xb856e0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0043d3300, 0x75a856, 0x0, 0xc000978460, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0043d3300, 0xc0043d32c8, 0xc0043d3350, 0x42ca96, 0xc001a6e150, 0x30)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xaba6c1, 0x1a, 0x1, 0xc0043d3368, 0xc0043d3380, 0x42d308, 0x30)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xaba6c1, 0x1a, 0x1, 0xb856e0, 0xc00608c8c0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0012bae70, 0xaba6c1, 0x1a, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc0043d3518, 0xc001c5e9e0, 0xaba6c1, 0x1a, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:134 +0x401
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1258 [semacquire]:
strings.makeCutsetFunc(0xaa4e4b, 0x5, 0xa7c820)
/usr/local/go/src/strings/strings.go:830 +0x4a
strings.TrimLeft(0xc0029720a8, 0x5, 0xaa4e4b, 0x5, 0x5, 0x0)
/usr/local/go/src/strings/strings.go:855 +0x61
golang.org/x/net/html.inHeadIM(0xc001c7e340, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:626 +0x969
golang.org/x/net/html.(*parser).parseCurrentToken(0xc001c7e340)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc001c7e340, 0xc002c86fc0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7b620, 0xc004c1f350, 0x0, 0x0, 0x0, 0x0, 0xa01, 0xc004c1f2f0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc004c1f2f0, 0x0, 0x0, 0x0, 0xc00, 0xc002baa800, 0xa84, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:93 +0x76
github.com/jaytaylor/html2text.FromString(0xc002baa800, 0xa84, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf63ae0900000000)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc004313518, 0xc00115b0e0, 0xc005bce860)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8780, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1259 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc0060310c0, 0xc0027bb600, 0x148, 0x148, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:430 +0x233
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0060310c0, 0xc0006015e0, 0xc00602fe48, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060310c0, 0xc000601570, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0060310c0, 0xc000601570, 0xc000000001, 0x9)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0060310c0, 0xc000601570, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0060310c0, 0xc000601570, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060310c0, 0xc000601490, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0060310c0, 0xc000601490, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0060310c0, 0xc000601490, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060310c0, 0xc0052ff960, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00070b0c0, 0xc0052ff960, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00070b0c0, 0xc0052ff960, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060310c0, 0xc00193df80, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00070b0c0, 0xc00193df80, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00070b0c0, 0xc00193df80, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060310c0, 0xc000221420, 0x7fe23811c830, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00070b0c0, 0xc000221420, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00070b0c0, 0xc000221420, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0060310c0, 0xc000221340, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00070b0c0, 0xc000221340, 0x54b170, 0xc0001985b0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc000221340, 0x0, 0x0, 0x0, 0x0, 0xc000221340, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc002c70750, 0x0, 0x0, 0x0, 0x1300, 0xc0046b9900, 0x1042, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0046b9900, 0x1042, 0x0, 0x0, 0x0, 0x0, 0xad060b, 0x3, 0xc0041a5210)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc00070b518, 0xc005048ca0, 0xc00039e920)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8900, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1260 [semacquire]:
bytes.makeSlice(0xc63, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc001098f30, 0x105, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc001098f30, 0xc002ca6900, 0x105, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc001098f30, 0xc002ca6900, 0x105, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc001098f30, 0xc0044c7260, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc001098f30, 0xc0044c71f0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001098f30, 0xc0044c6a80, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001098f30, 0xc0044c69a0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc001098f30, 0xc0044c62a0, 0xb84d20, 0xc001098f30)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc001098f30, 0xc0044c62a0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0050320c0, 0x19479634b, 0x1251a20, 0xc001000f60, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8e40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1261 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc00649a390, 0x1640f350d, 0x1251a20, 0xc003382a80, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f8fc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1262 [semacquire]:
github.com/jaytaylor/html2text.FromString(0xc002414a00, 0x13cd, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3aa7de4d00000000)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:102 +0x42
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003815518, 0xc00656c2e0, 0xc005ca6c80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9140, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1263 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003bab3c0, 0xaa04f9, 0x2, 0xc006304800, 0xb3d, 0x1, 0x23b, 0xc00656dd80)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc006304800, 0xb3d, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003bab518, 0xc0033831e0, 0x3a, 0xc0043973c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f92c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1264 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc004939860, 0x20, 0x203001, 0x203001, 0x203001)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003bad518, 0xc00115b4a0, 0xc005bced00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:285 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9440, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1265 [semacquire]:
regexp.(*machine).alloc(...)
/usr/local/go/src/regexp/exec.go:110
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x46, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:363 +0x46c
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x4c, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x52, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x59, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x60, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x71, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x7c, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x84, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x8d, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x9d, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0xa7, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0xc2, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0xd4, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0xdc, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0xe2, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x10d, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x117, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x121, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x12a, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x131, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x18a, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x195, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x1b1, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x1c3, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x1c9, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x1db, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x1e2, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x1fa, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x202, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x20e, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x217, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x221, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x227, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x231, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x247, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x251, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x258, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x270, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x27b, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x284, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x28b, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x293, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x29b, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x2a8, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x2d0, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x2e8, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x2ef, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x2fa, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x305, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x30e, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x316, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x327, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x330, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x33e, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x344, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x352, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x36a, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x376, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0xc0022c0680)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0xc000000382, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x60)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x38c, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0xc001effb60)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x3a0, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0xc0022c0828)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x3af, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0xc00394fc00)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x3c3, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x101000100)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x3ca, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0xab52f4)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x3d3, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0xab6d98)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x3db, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x13)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x3e1, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x3e8, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x3f3, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0xc0000003fd, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x404, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x415, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0xc00038e060)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x41f, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x426, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x43b, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x44c, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x46d, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0xc000000475, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x485, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x48d, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x49c, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x203000)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x4a3, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x4ad, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x7fe23a6bafff)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x4c2, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0xc001883080)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0xc0000004dd, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0xc000b71917)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0xc0000004e6, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0xab47fe)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x501, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x8bf0000051d, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0xc0022c1ab8)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x52b, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x42ca96)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0xc000000538, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x9ed640)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x579, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0xc0065a9e20)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0x58e, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x1189d15)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0xc000000598, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x1189cea)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0xc0000005a1, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
regexp.(*machine).add(0xc0014e23c0, 0xc0014e23d0, 0xc0000005a9, 0x0, 0xc0042c42a0, 0x0, 0x2, 0xc0067a2f58, 0x0, 0x0)
/usr/local/go/src/regexp/exec.go:340 +0x2ec
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1266 [semacquire]:
bytes.makeSlice(0xd14, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0019f5138, 0xd6, 0x13d)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0019f5138, 0xc004900460, 0xd6, 0x5165d7, 0xc0044c5688, 0xc0049108c8)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00491f6c0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00491f650)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc00431ad90)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc0049a3710, 0xaa03ab, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).isHighLinkDensity(0xc0019f5518, 0xc0049a3710, 0x2)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:566 +0x95
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0019f5518, 0xc00229cea0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:386 +0x1ae
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9740, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1268 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc006596750, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc006596820, 0xc0065c6070, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0023004e0, 0x0, 0x0, 0x0, 0xc0045ab200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0023004e0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0045ab8f8, 0xc0036b0000, 0x148f, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9a40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1269 [semacquire]:
golang.org/x/net/html.(*parser).addChild(0xc000386680, 0xc000ce2000)
/home/develop/go/src/golang.org/x/net/html/parse.go:234 +0xf3
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc000386680, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:1097 +0x76a
golang.org/x/net/html.(*parser).parseCurrentToken(0xc000386680)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc000386680, 0xc001cf7260, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00103e940, 0x0, 0x0, 0x0, 0xc005417200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00103e940, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0054178f8, 0xc002d48000, 0xbac, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9bc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1270 [semacquire]:
regexp.(*Regexp).get(0xc000149cc0, 0xc000382801)
/usr/local/go/src/regexp/regexp.go:256 +0x322
regexp.(*Regexp).doExecute(0xc000149cc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0065637d0, 0x9, 0x0, 0x0, ...)
/usr/local/go/src/regexp/exec.go:538 +0xb5
regexp.(*Regexp).doMatch(...)
/usr/local/go/src/regexp/exec.go:514
regexp.(*Regexp).MatchString(...)
/usr/local/go/src/regexp/regexp.go:507
github.com/advancedlogic/GoOse.(*Cleaner).cleanBadTags.func1.1(0x0, 0xc002c9c6f0)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:427 +0xd0
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc002c9c690, 0xc00173f250, 0x7)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).cleanBadTags.func1(0x1, 0xc002c9c600)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:425 +0x17d
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc002c9c510, 0xc00173f2f8, 0x4)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).cleanBadTags(0xc00173f6b0, 0xc002489e80, 0xc000149cc0, 0xc00173f3a0, 0xc006087020)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:421 +0xa4
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc00173f6b0, 0xc002489e80, 0xc0059f4820)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:293 +0x199
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9e00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1271 [runnable]:
regexp.(*machine).step(0xc001e1a000, 0xc001e1a010, 0xc001e1a040, 0x23a, 0x23b, 0x65, 0xc004a8f7f0)
/usr/local/go/src/regexp/exec.go:260 +0x6d0
regexp.(*machine).match(0xc001e1a000, 0xb886e0, 0xc001e1a0c0, 0x23a, 0xc004a8f8f0)
/usr/local/go/src/regexp/exec.go:225 +0x21f
regexp.(*Regexp).doExecute(0xc000149ae0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0005e7500, 0x1407, 0x1ba, 0x2, ...)
/usr/local/go/src/regexp/exec.go:542 +0x177
regexp.(*Regexp).replaceAll(0xc000149ae0, 0x0, 0x0, 0x0, 0xc0005e7500, 0x1407, 0x2, 0xc004a8fa58, 0xaa3a2a, 0x4, ...)
/usr/local/go/src/regexp/regexp.go:599 +0x1a5
regexp.(*Regexp).ReplaceAllString(0xc000149ae0, 0xc0005e7500, 0x1407, 0x0, 0x0, 0xc00038e060, 0x127d7a0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
_/home/ec2-user/environment/cluster2/cluster2/base.Sanitize(0xaa3a2a, 0x4, 0xc0005e7500, 0x1407, 0x0, 0x0, 0x10, 0xab52f4)
/home/ec2-user/environment/cluster2/cluster2/base/sanitizer.go:44 +0xc9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0015f9f80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:304 +0x33a
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1272 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00039fe00, 0xc0045a72c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0045a73c0, 0xaa058b, 0x2, 0xc006069c00, 0x683, 0x7, 0x16b, 0xc00103f580)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc006069c00, 0x683, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0045a7518, 0xc001885d80, 0x3b, 0xc002177380)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003806c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1273 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc003ceedd0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:901 +0x820
golang.org/x/net/html.(*parser).parseCurrentToken(0xc003ceedd0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc003ceedd0, 0xc0062e12d0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc002489340, 0x0, 0x0, 0x0, 0xc002b9b200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc002489340, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc002b9b8f8, 0xc00062e480, 0x1a0a, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003816e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1274 [runnable]:
sync.(*RWMutex).RUnlock(0xc00039e088)
/usr/local/go/src/sync/rwmutex.go:62 +0x50
github.com/fatih/set.(*Set).Has(0xc00039e080, 0xc00693d220, 0x1, 0x1, 0xc000480001)
/home/develop/go/src/github.com/fatih/set/set_ts.go:86 +0x10c
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00693d588, 0xaa04f9, 0x2, 0xc0017bc700, 0x64d, 0x0, 0xc003be6380, 0x7fe260fed560)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:81 +0x1e7
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00693d518, 0xc000b32ba0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000381800, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1275 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc00092e000, 0xf90, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:81 +0x208
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc00092e000, 0xf90, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc00092e000, 0xf90, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc00092e000, 0xf90, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc00092e000, 0xf90, 0x0, 0x0, 0xf90, 0xc00092e000, 0xf90)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc00092e000, 0xf90, 0xf90, 0x1000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000a14cfa, 0x22, 0xc00092e000, 0xf90, 0xc00092e000, 0xf90, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000a14cfa, 0x22, 0xc00092e000, 0xf90, 0xaa0378, 0x1, 0xc00092e000, 0xf90, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003818c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1276 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc004e4d0c0, 0xc001d773f1, 0xa6, 0xa6, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:430 +0x233
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc004e4d0c0, 0xc00474d110, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc004e4d0c0, 0xc00474cf50, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc004e4d0c0, 0xc00474cf50, 0xc000000001, 0x7)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc004e4d0c0, 0xc00474cf50, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc004e4d0c0, 0xc00474cf50, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc004e4d0c0, 0xc00474c930, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc003bfd0c0, 0xc00474c930, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc003bfd0c0, 0xc00474c930, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc004e4d0c0, 0xc00474c850, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc003bfd0c0, 0xc00474c850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc003bfd0c0, 0xc00474c850, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc004e4d0c0, 0xc00474c150, 0x8, 0x45624c)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc003bfd0c0, 0xc00474c150, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc003bfd0c0, 0xc00474c150, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc004e4d0c0, 0xc00474c070, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc003bfd0c0, 0xc00474c070, 0x54b170, 0xc0035ef520)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc00474c070, 0x0, 0x0, 0x0, 0x0, 0xc00474c070, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc00229be30, 0x0, 0x0, 0x0, 0x1500, 0xc0043ef500, 0x1454, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0043ef500, 0x1454, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000ccc4c0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc003bfd518, 0xc00135ed80, 0xc0034d0c20)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000381aa0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1277 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc002eec340, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x18a
golang.org/x/net/html.(*parser).parse(0xc002eec410, 0xc0044c75e0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0027ee180, 0x0, 0x0, 0x0, 0xc000685200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0027ee180, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0006858f8, 0xc0019a8500, 0x20cc, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000381ec0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1279 [runnable]:
bytes.(*Reader).Read(0xc005bb4f60, 0xc0069a2600, 0x800, 0x800, 0x400, 0x0, 0x0)
/usr/local/go/src/bytes/reader.go:40 +0xf3
bytes.(*Buffer).ReadFrom(0xc0048c90e8, 0xb7b620, 0xc005bb4f60, 0x203001, 0x203001, 0x203001)
/usr/local/go/src/bytes/buffer.go:204 +0xb1
io/ioutil.readAll(0xb7b620, 0xc005bb4f60, 0x200, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/io/ioutil/ioutil.go:36 +0xe3
io/ioutil.ReadAll(...)
/usr/local/go/src/io/ioutil/ioutil.go:45
github.com/ssor/bom.NewReaderWithoutBom(0xb7b620, 0xc005bb4f60, 0xc0048c91d8, 0x42d308, 0x30, 0xa4a0e0)
/home/develop/go/src/github.com/ssor/bom/bom.go:29 +0x42
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc005bb4f60, 0x0, 0x0, 0x0, 0x1000, 0xc003138000, 0xe72, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:89 +0x39
github.com/jaytaylor/html2text.FromString(0xc003138000, 0xe72, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc000c51780)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0048c9518, 0xc00135e3e0, 0xc004c38680)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000478300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1280 [semacquire]:
bytes.makeSlice(0x1c2d, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0028c6cc0, 0x97, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0028c6cc0, 0xc0009d5f40, 0x97, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc0028c6cc0, 0xc0009d5f40, 0x97, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc0028c6cc0, 0xc0048d8f50, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc0028c6cc0, 0xc0048d8ee0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc0028c6cc0, 0xc0048d84d0, 0xb84d20, 0xc0028c6cc0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc0028c6cc0, 0xc0048d84d0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0028c6c00, 0x203000, 0x203000, 0x7fe2380c7778, 0x40)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs.func1(0x0, 0xc0028c6c00)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:496 +0x67
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc0028c6ba0, 0xc0009432c8, 0x7)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs(0xc0009436b0, 0xc00135e780, 0xaac03e, 0x7, 0xc00135e780)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:495 +0xd8
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc0009436b0, 0xc00135e780, 0xc002301b60)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:302 +0x39d
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0004786c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1281 [semacquire]:
bytes.makeSlice(0x1da0, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc005946420, 0x432, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc005946420, 0xc004237680, 0x432, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc005946420, 0xc004237680, 0x432, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc005946420, 0xc005974070, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc005946420, 0xc0009378f0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc005946420, 0xc0009369a0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc005946420, 0xc0009368c0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc005946420, 0xc0009361c0, 0xb84d20, 0xc005946420)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc005946420, 0xc0009361c0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc005946270, 0x17f35a776, 0x1251a20, 0xc0009c6d00, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4000, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1282 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc0006424e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x18a
golang.org/x/net/html.(*parser).parse(0xc0006425b0, 0xc000311340, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc002409de0, 0x0, 0x0, 0x0, 0xc0016a5200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc002409de0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0016a58f8, 0xc003630000, 0x189c, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec40c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1283 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0034c7880, 0x345, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0034c7880, 0x345, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0034c7880, 0x345, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0034c7880, 0x345, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0034c7880, 0x345, 0x0, 0x0, 0x345, 0xc0034c7880, 0x345)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0034c7880, 0x345, 0x345, 0x380)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002b7cec0, 0x38, 0xc0034c7880, 0x345, 0xc0034c7880, 0x345, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002b7cec0, 0x38, 0xc0034c7880, 0x345, 0xaa0378, 0x1, 0xc0034c7880, 0x345, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4180, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1284 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc000387a00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc000387ad0, 0xc0000fddc0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0000b2900, 0x0, 0x0, 0x0, 0xc003f9f200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0000b2900, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc003f9f8f8, 0xc003dd8800, 0x1775, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4240, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1285 [runnable]:
strings.genSplit(0xc0034a6757, 0x296, 0xaa0378, 0x1, 0x0, 0x38a, 0xc0069ce000, 0x38b, 0x38b)
/usr/local/go/src/strings/strings.go:264 +0x133
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001f373c0, 0xaa058b, 0x2, 0xc0007d8c00, 0x10ed, 0x7, 0x38b, 0xc005048400)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0007d8c00, 0x10ed, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001f37518, 0xc005c55240, 0x39, 0xc00600b080)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1286 [semacquire]:
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc00448f2e0, 0x432e00, 0xc00448f210, 0x9a030e24, 0x78)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:724 +0x716
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc00448f2e0, 0xc0006ded80, 0x5, 0xc00448f230, 0x46620f)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc00448f2e0, 0xc00448f2f8, 0x43b3b7, 0x1252790, 0xc00448f200, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xaa64ee, 0x5, 0x7fe23805a8f8, 0xc00448f301, 0x10100000042ab7a, 0x0, 0x826)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xaa64ee, 0x5, 0x0, 0x0, 0xaa0301)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc0032322d0, 0xaa64ee, 0x5, 0xb6c)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).getTitleUnmodified(0xc00448f518, 0xc00137e300, 0x826, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:57 +0x47
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:128 +0x282
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec43c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1287 [semacquire]:
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0045b4ff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:766 +0x1e7
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0045b4ff8, 0x1, 0xc001c2efc0, 0xc01009e5100, 0x280460fef080)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0045b4ff8, 0x4, 0x607, 0xc00110b901, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc00110b000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc002872b60, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc00118acc0, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0045b58f8, 0xc0044faa80, 0x9f7, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1288 [semacquire]:
bytes.makeSlice(0x1af4, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc002c8f230, 0x122, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc002c8f230, 0xc0046fa9c3, 0x122, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc002c8f230, 0xc0046fa9c3, 0x145, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc002c8f230, 0xc0026febd0, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc002c8f230, 0xc0026feb60, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc002c8f230, 0xc0026fe930, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc002c8f230, 0xc000565a40, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc002c8f230, 0xc000565960, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc002c8f230, 0xc000565260, 0xb84d20, 0xc002c8f230)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc002c8f230, 0xc000565260, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc002c8f020, 0x1720fbeb8, 0x1251a20, 0xc005bcf500, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4540, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1289 [semacquire]:
strings.genSplit(0xc004c8c800, 0x72f, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0037f8600, 0x181, 0x181)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc003fa13c0, 0xaa059b, 0x2, 0xc002de6800, 0x72f, 0x16, 0x181, 0xc00608c560)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002de6800, 0x72f, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003fa1518, 0xc0000b33e0, 0x35, 0xc001286d80)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1290 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2340a21e0, 0x7fe1f41d3ea0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00322b3f0, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00322b3e0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002d35180, 0x348, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000b6c540, 0x2f, 0xc002d35180, 0x348, 0xc002d35180, 0x348, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000b6c540, 0x2f, 0xc002d35180, 0x348, 0xaa0378, 0x1, 0xc002d35180, 0x348, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec46c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1291 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc005e1f800, 0xb46, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc005e1f800, 0xb46, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc005e1f800, 0xb46, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc005e1f800, 0xb46, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc005e1f800, 0xb46, 0x0, 0x0, 0xb46, 0xc005e1f800, 0xb46)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc005e1f800, 0xb46, 0xb46, 0xc80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc00112fe80, 0x37, 0xc005e1f800, 0xb46, 0xc005e1f800, 0xb46, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc00112fe80, 0x37, 0xc005e1f800, 0xb46, 0xaa0378, 0x1, 0xc005e1f800, 0xb46, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4780, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1292 [semacquire]:
bytes.makeSlice(0x956, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc005186ea0, 0x1b0, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc005186ea0, 0xc000595c00, 0x1b0, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc005186ea0, 0xc000595c00, 0x1b0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc005186ea0, 0xc0049deaf0, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc005186ea0, 0xc0049dea80, 0xb84d20, 0xc005186ea0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc005186ea0, 0xc0049dea80, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc005186e70, 0x203001, 0x203001, 0xac16eaef00000000, 0xc005f07138)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs.func1(0x0, 0xc005186e70)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:496 +0x67
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc005186e40, 0xc0011872c8, 0x7)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).convertDivsToParagraphs(0xc0011876b0, 0xc0032a8740, 0xaac03e, 0x7, 0xc0032a8740)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:495 +0xd8
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc0011876b0, 0xc0032a8740, 0xc005bcf460)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:302 +0x39d
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4840, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1293 [semacquire]:
regexp.(*Regexp).get(0xc000149cc0, 0xc000622b01)
/usr/local/go/src/regexp/regexp.go:256 +0x2f5
regexp.(*Regexp).doExecute(0xc000149cc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc0049e2720, 0x9, 0x0, 0x0, ...)
/usr/local/go/src/regexp/exec.go:538 +0xb5
regexp.(*Regexp).doMatch(...)
/usr/local/go/src/regexp/exec.go:514
regexp.(*Regexp).MatchString(...)
/usr/local/go/src/regexp/regexp.go:507
github.com/advancedlogic/GoOse.(*Cleaner).cleanBadTags.func1.1(0x0, 0xc002d00570)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:427 +0xd0
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc002d00540, 0xc0045d5250, 0x7)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).cleanBadTags.func1(0x1, 0xc002d00480)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:425 +0x17d
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc002d00390, 0xc0045d52f8, 0x4)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*Cleaner).cleanBadTags(0xc0045d56b0, 0xc00103e1c0, 0xc000149cc0, 0xc0045d53a0, 0xc002c12240)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:421 +0xa4
github.com/advancedlogic/GoOse.(*Cleaner).Clean(0xc0045d56b0, 0xc00103e1c0, 0xc002f71f60)
/home/develop/go/src/github.com/advancedlogic/GoOse/cleaner.go:293 +0x199
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:149 +0x614
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4900, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1294 [runnable]:
strings.genSplit(0xc005da2488, 0xafe, 0xaa0378, 0x1, 0x0, 0x252, 0x42fa20, 0x253, 0x253)
/usr/local/go/src/strings/strings.go:264 +0x133
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc004c073c0, 0xaa0503, 0x2, 0xc005da0000, 0xb86, 0x6, 0x253, 0xc0009c7c60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc005da0000, 0xb86, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004c07518, 0xc0009c7840, 0x39, 0xc002b7ca40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec49c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1297 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe11024e8e0, 0x7fe170249dc0)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00162b850, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00162b840, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0007f8a80, 0xa73, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0033549da, 0x22, 0xc0007f8a80, 0xa73, 0xc0007f8a80, 0xa73, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0033549da, 0x22, 0xc0007f8a80, 0xa73, 0xaa0378, 0x1, 0xc0007f8a80, 0xa73, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4c00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1298 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc00608c4e0, 0xc0045af2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0045af3c0, 0xaa0599, 0x2, 0xc0027ac600, 0x101f, 0xc0027ac600, 0xc0045af3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0027ac600, 0x101f, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0045af518, 0xc002f71920, 0x3b, 0xc003355800)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4cc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1299 [semacquire]:
bytes.makeSlice(0x22a, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0036f3388, 0xd0, 0x1)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0036f3388, 0xc00027e410, 0xd0, 0x5165d7, 0xc005d02ef1, 0xc0025145c6)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc002601f10)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:70 +0x8d
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc002601ea0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc002601880)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0026017a0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text.func1(0xc0026010a0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:74 +0x55
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc0041f24b0, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:79 +0xbd
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0036f3518, 0xc000763580, 0x2d, 0xc000a14120)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4d80, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1300 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc0026fb1e0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:901 +0x820
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0026fb1e0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0026fb1e0, 0xc004201810, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc002f71ba0, 0x0, 0x0, 0x0, 0xc000893200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc002f71ba0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0008938f8, 0xc000df6000, 0xddd, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4e40, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1301 [runnable]:
strings.Index(0xc006453c20, 0x11b, 0xaa0378, 0x1, 0x29)
/usr/local/go/src/strings/strings.go:1028 +0x67a
strings.genSplit(0xc006453c20, 0x11b, 0xaa0378, 0x1, 0x0, 0x29, 0x11b, 0xc006453b00, 0x11b)
/usr/local/go/src/strings/strings.go:260 +0xf3
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001b1f588, 0xaa04f9, 0x2, 0xc006453b00, 0x11b, 0x5, 0x2a, 0xc0033831c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc001b1f518, 0xc004938e10, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:528 +0xfd
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc001b1f518, 0xc00522c5c0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4f00, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1302 [semacquire]:
strings.genSplit(0xc004318000, 0xa6d, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x0, 0xc00351af50, 0x20000c0040dd360)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0040dd3c0, 0xaa04f9, 0x2, 0xc003f6b500, 0xa6d, 0xc003f6b500, 0xc0040dd3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003f6b500, 0xa6d, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0040dd518, 0xc001396680, 0x26, 0xc000a141b0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec4fc0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1303 [semacquire]:
bytes.makeSlice(0x221f, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc0029214a0, 0x683, 0x2)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc0029214a0, 0xc001aa2000, 0x683, 0x1, 0x127e7a0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.render1(0xb84d20, 0xc0029214a0, 0xc004407960, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:202 +0x694
golang.org/x/net/html.render1(0xb84d20, 0xc0029214a0, 0xc004407880, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc0029214a0, 0xc004406d20, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc0029214a0, 0xc004406c40, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc0029214a0, 0xc00451fce0, 0xb84d20, 0xc0029214a0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc0029214a0, 0xc00451fce0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc002921290, 0xf0389c14, 0x1251a20, 0xc000762920, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5080, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1304 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc002658000, 0x15d1, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc002658000, 0x15d1, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc002658000, 0x15d1, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc002658000, 0x15d1, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc002658000, 0x15d1, 0x0, 0x0, 0x15d1, 0xc002658000, 0x15d1)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc002658000, 0x15d1, 0x15d1, 0x1800)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0005cda40, 0x19, 0xc002658000, 0x15d1, 0xc002658000, 0x15d1, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0005cda40, 0x19, 0xc002658000, 0x15d1, 0xaa0378, 0x1, 0xc002658000, 0x15d1, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5140, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1306 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe12c00dec0, 0x7fe12c00e060)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc003c73e00, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc003c73df0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001386a00, 0x69f, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc001bfbdc0, 0x33, 0xc001386a00, 0x69f, 0xc001386a00, 0x69f, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc001bfbdc0, 0x33, 0xc001386a00, 0x69f, 0xaa0378, 0x1, 0xc001386a00, 0x69f, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec52c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1307 [runnable]:
sync.(*RWMutex).Unlock(0xc00642bb88)
/usr/local/go/src/sync/rwmutex.go:118 +0xd2
github.com/fatih/set.(*Set).Add(0xc00642bb80, 0xc004df3220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:37 +0xfc
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc004df3588, 0xaa04f9, 0x2, 0xc00587f000, 0xf2d, 0xd, 0x19, 0xc00642bb40)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc004df3518, 0xc0013968e0, 0x4f)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5380, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1308 [semacquire]:
strings.Replace(0xc00410b600, 0x80b, 0xaa3f06, 0x4, 0xaa1c36, 0x3, 0x2, 0xc00410b600, 0x80b)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002049518, 0xc001397460, 0xc00522d0c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:316 +0x67b
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5440, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1309 [semacquire]:
strings.genSplit(0xc00319e000, 0x9c4, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0032e0500, 0x21f, 0x21f)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0045ad3c0, 0xaa04fd, 0x2, 0xc003736a80, 0x9c4, 0x1, 0x21f, 0xc0018842c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc003736a80, 0x9c4, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0045ad518, 0xc003382220, 0x54, 0xc0006dde00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5500, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1310 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc000389b80, 0x514, 0xc0040db218, 0xc001d7aa80)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc000389b80, 0x514, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0040db3c0, 0xaa0599, 0x2, 0xc000389b80, 0x514, 0x1, 0x132, 0xc000feee80)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000389b80, 0x514, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0040db518, 0xc000762300, 0x36, 0xc00112e4c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec55c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1311 [semacquire]:
strings.genSplit(0xc004db5900, 0x1070, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc004fd5800, 0x379, 0x379)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0020453c0, 0xaa04fd, 0x2, 0xc001264c00, 0x1070, 0x1, 0x379, 0xc0034d0b60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001264c00, 0x1070, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002045518, 0xc001397e80, 0x53, 0xc00271e600)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5680, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1312 [semacquire]:
strings.genSplit(0xc00441f200, 0x5f5, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x0, 0xc006064e70, 0x20000c003819360)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0038193c0, 0xaa04f9, 0x2, 0xc00441ec00, 0x5f5, 0xc00441ec00, 0xc0038193b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00441ec00, 0x5f5, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003819518, 0xc001cfca00, 0x3a, 0xc000b736c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5740, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1313 [runnable]:
strings.Index(0xc0065bd275, 0xc83, 0xaa0378, 0x1, 0x0)
/usr/local/go/src/strings/strings.go:1028 +0x67a
strings.genSplit(0xc0065bd275, 0xc83, 0xaa0378, 0x1, 0x0, 0x2dd, 0x42fa20, 0x2de, 0x2de)
/usr/local/go/src/strings/strings.go:260 +0xf3
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000b313c0, 0xaa0569, 0x2, 0xc001fc8000, 0xef8, 0x46, 0x2de, 0xc005bce220)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc001fc8000, 0xef8, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000b31518, 0xc002f70080, 0x5e, 0xc002b2a30a)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5800, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1314 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1981d4650, 0x7fe1981d4670)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0033a5d30, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0033a5d20, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc003774700, 0x615, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002eea550, 0x48, 0xc003774700, 0x615, 0xc003774700, 0x615, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002eea550, 0x48, 0xc003774700, 0x615, 0xaa0378, 0x1, 0xc003774700, 0x615, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec58c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1315 [semacquire]:
bytes.makeSlice(0x193f, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00291d1a0, 0xf5, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00291d1a0, 0xc002ab5696, 0xf5, 0x6, 0xf5, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc00291d1a0, 0xc002ab5696, 0x122, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:201 +0xe5
golang.org/x/net/html.render1(0xb84d20, 0xc00291d1a0, 0xc00226ee00, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc00291d1a0, 0xc00226ed90, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00291d1a0, 0xc002af7ab0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00291d1a0, 0xc002af79d0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00291d1a0, 0xc004407ab0, 0xb84d20, 0xc00291d1a0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc00291d1a0, 0xc004407ab0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0029217d0, 0x17156a741, 0x1251a20, 0xc000762ec0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5980, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1316 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc004d11000, 0xeb9, 0xc002f6b218, 0xc001d7aa80)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc004d11000, 0xeb9, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002f6b3c0, 0xaa0503, 0x2, 0xc004d11000, 0xeb9, 0x5, 0x319, 0xc00608cd40)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004d11000, 0xeb9, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002f6b518, 0xc005a65400, 0x57, 0xc003d602b6)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5a40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1317 [semacquire]:
strings.genSplit(0xc0044c2700, 0x665, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc0037f5300, 0x183, 0x183)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00088f3c0, 0xaa0599, 0x2, 0xc0017bf100, 0x665, 0x1, 0x183, 0xc005a65a40)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0017bf100, 0x665, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00088f518, 0xc0013969a0, 0x46, 0xc000bbb450)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5b00, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1318 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc000bd8340, 0xc00552a000, 0x70)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc000bd8340, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc000bd8340)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc000bd8340, 0xc0005d2af0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00115adc0, 0x0, 0x0, 0x0, 0xc0044db200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00115adc0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0044db8f8, 0xc00329a000, 0x211d, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5bc0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1319 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc0065cf500, 0x1401, 0xc001ad1218, 0xc001d7aa80)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc0065cf500, 0x1401, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001ad13c0, 0xaa058b, 0x2, 0xc0065cf500, 0x1401, 0x5, 0x4bc, 0xc001885940)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0065cf500, 0x1401, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc001ad1518, 0xc004c39b60, 0x10, 0xc002515b50)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5c80, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1320 [semacquire]:
regexp/syntax.(*compiler).inst(...)
/usr/local/go/src/regexp/syntax/compile.go:173
regexp/syntax.(*compiler).rune(0xc0004b6b30, 0xc00600bf58, 0x1, 0xa, 0xd5, 0xe00000007)
/usr/local/go/src/regexp/syntax/compile.go:267 +0x27c
regexp/syntax.(*compiler).compile(0xc0004b6b30, 0xc0027e2ee0, 0x200000001)
/usr/local/go/src/regexp/syntax/compile.go:110 +0x12c
regexp/syntax.(*compiler).compile(0xc0004b6b30, 0xc0027e2f50, 0x0)
/usr/local/go/src/regexp/syntax/compile.go:156 +0xb29
regexp/syntax.Compile(0xc0027e2f50, 0xc0027e2f50, 0x1, 0x1)
/usr/local/go/src/regexp/syntax/compile.go:83 +0x10a
regexp.compile(0xac0e09, 0x13, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:178 +0xd7
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc0004b6ff8, 0x0, 0xac0dfd, 0xa)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc0004b6ff8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0xc002515e80)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0004b6ff8, 0x1189c00, 0x1, 0x1, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0004b6ff8, 0x1, 0xc005f2a550, 0xc0009e5180, 0x280460fef080)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0004b6ff8, 0x4, 0xcbb, 0xc003639301, 0x115d6b1300000004, 0x5c50400005604)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xac0df8, 0x25, 0xc003639000, 0x1000, 0x1000, 0x0, 0x8)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xac0df8, 0x25, 0x203000, 0xc000643110, 0x1)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc002c8f8c0, 0xac0df8, 0x25, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.Crawler.GetContentType(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:43 +0x52
github.com/advancedlogic/GoOse.Crawler.GetCharset(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:57 +0x89
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0041b78f8, 0xc0032a3400, 0x22d3, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:93 +0x1d6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5d40, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1321 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc000bd8680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1181 +0x18a
golang.org/x/net/html.(*parser).parse(0xc000bd8750, 0xc00068a700, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00115ae60, 0x0, 0x0, 0x0, 0xc004315200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00115ae60, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0043158f8, 0xc003428c00, 0x2571, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5e00, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1322 [runnable]:
github.com/fatih/set.(*Set).Has(0xc00039e200, 0xc00401d2c0, 0x1, 0x1, 0xc000480000)
/home/develop/go/src/github.com/fatih/set/set_ts.go:82 +0xb5
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00401d3c0, 0xaa04d9, 0x2, 0xc0026bd000, 0x2e27, 0xc0026bd000, 0xc00401d3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:81 +0x1e7
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0026bd000, 0x2e27, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00401d518, 0xc002408300, 0x3c, 0xc002b7d2c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5ec0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1323 [semacquire]:
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc000a592e0, 0x432e00, 0xc000a59210, 0x9a030e24, 0x131)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:724 +0x716
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc000a592e0, 0xc000454300, 0x5, 0xc000a59230, 0x46620f)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc000a592e0, 0xc000a592f8, 0x43b3b7, 0x1252790, 0xc000a59200, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xaa64ee, 0x5, 0x7fe238052f30, 0xc000a59301, 0x10100000042ab7a, 0x0, 0xabc)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xaa64ee, 0x5, 0x0, 0x0, 0xaa0301)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc00229b0e0, 0xaa64ee, 0x5, 0xb82)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).getTitleUnmodified(0xc000a59518, 0xc00115b020, 0xabc, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:57 +0x47
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:128 +0x282
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc003ec5f80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1324 [semacquire]:
strings.Replace(0xc004b86000, 0x1043, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0xe, 0xc004b86000, 0x1043)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0010b98f8, 0xc004549900, 0x1041, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d2060, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1325 [semacquire]:
regexp.(*Regexp).replaceAll(0xc000149ae0, 0x0, 0x0, 0x0, 0xc005819502, 0x146d, 0x2, 0xc0010b7a58, 0xaa3a2a, 0x4, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc000149ae0, 0xc005819502, 0x146d, 0x0, 0x0, 0xc00038e060, 0x127d7a0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
_/home/ec2-user/environment/cluster2/cluster2/base.Sanitize(0xaa3a2a, 0x4, 0xc005819502, 0x146d, 0x0, 0x0, 0x10, 0xab52f4)
/home/ec2-user/environment/cluster2/cluster2/base/sanitizer.go:44 +0xc9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d2120, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:304 +0x33a
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1326 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0037381b0, 0x171666cf7, 0x1251a20, 0xc000763320, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:109 +0x113
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d21e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1327 [semacquire]:
bytes.makeSlice(0xf60, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc005bb32c0, 0x20a, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc005bb32c0, 0xc004039d40, 0x20a, 0x6, 0xffffffffffffffff, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc005bb32c0, 0xc004039d40, 0x20a, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc005bb32c0, 0xc00079f7a0, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc005bb32c0, 0xc00079f730, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc005bb32c0, 0xc00079ebd0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc005bb32c0, 0xc00079eaf0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc005bb32c0, 0xc00079e3f0, 0xb84d20, 0xc005bb32c0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc005bb32c0, 0xc00079e3f0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc005bb30e0, 0x18f74a27f, 0x1251a20, 0xc00642b700, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d22a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1328 [runnable]:
golang.org/x/net/html.(*Tokenizer).readByte(0xc0029ec820, 0x64)
/home/develop/go/src/golang.org/x/net/html/token.go:234 +0x440
golang.org/x/net/html.(*Tokenizer).Next(0xc0029ec820, 0x10100000004)
/home/develop/go/src/golang.org/x/net/html/token.go:980 +0xcd
golang.org/x/net/html.(*parser).parse(0xc0029ec8f0, 0xc00079fd50, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2284 +0xad
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0034d1ea0, 0x0, 0x0, 0x0, 0xc00460b200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0034d1ea0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00460b8f8, 0xc004294000, 0xf93, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d2360, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1329 [runnable]:
sync.(*RWMutex).Lock(0xc00642b988)
/usr/local/go/src/sync/rwmutex.go:92 +0xa3
github.com/fatih/set.(*Set).Add(0xc00642b980, 0xc0041352c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:31 +0x58
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0041353c0, 0xaa04fd, 0x2, 0xc004398000, 0xa8f, 0x1, 0x240, 0xc003383e60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc004398000, 0xa8f, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004135518, 0xc0034d10c0, 0x3d, 0xc000c50100)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d2420, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1330 [semacquire]:
strings.genSplit(0xc00376ec00, 0xba1, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc001e26000, 0x2a7, 0x2a7)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0022b93c0, 0xaa058b, 0x2, 0xc00051c800, 0xba1, 0x6, 0x2a7, 0xc00137e2a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00051c800, 0xba1, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0022b9518, 0xc000763380, 0x46, 0xc0026dc140)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d24e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1331 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc004070000, 0x453, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:83 +0x3e0
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc004070000, 0x453, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc004070000, 0x453, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc004070000, 0x453, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc004070000, 0x453, 0x0, 0x0, 0x453, 0xc004070000, 0x453)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc004070000, 0x453, 0x453, 0x480)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc000c531d0, 0x28, 0xc004070000, 0x453, 0xc004070000, 0x453, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc000c531d0, 0x28, 0xc004070000, 0x453, 0xaa0378, 0x1, 0xc004070000, 0x453, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d25a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1332 [semacquire]:
golang.org/x/net/html.NewTokenizerFragment(0xb7c280, 0xc005d7a2c0, 0x0, 0x0, 0x20)
/home/develop/go/src/golang.org/x/net/html/token.go:1213 +0x5d
golang.org/x/net/html.NewTokenizer(...)
/home/develop/go/src/golang.org/x/net/html/token.go:1201
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc005d7a2c0, 0x0, 0x0, 0x0, 0xc003eab200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2337 +0x41
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc005d7a2c0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc003eab8f8, 0xc002e70000, 0x1329, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d2660, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1333 [runnable]:
github.com/fatih/set.(*Set).Add(0xc001c5e4e0, 0xc000afd2c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000afd3c0, 0xaa0503, 0x2, 0xc006782900, 0x88f, 0x5, 0x1e2, 0xc001c5e4c0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc006782900, 0x88f, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000afd518, 0xc005d7a320, 0x4b, 0xc000607090)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d2720, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1334 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc005997cc0, 0x124, 0x124, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc005997cc0, 0x124, 0x7544a2, 0x0)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00158b588, 0xaa04f9, 0x2, 0xc005997cc0, 0x124, 0xa, 0x15, 0xc00656d720)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00158b518, 0xc003212660, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:385 +0x17f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d27e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1335 [semacquire]:
strings.Replace(0xc006570000, 0x1c3c, 0xaa042f, 0x2, 0xaa0af0, 0x3, 0x1, 0xc0064e6070, 0xc005d7a320)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:175 +0x72
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc002f4b8f8, 0xc006570000, 0x1c3c, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d28a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1336 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc000aa6d50, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002f8d518, 0xc0033835a0, 0x37, 0xc002177b40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d2960, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1337 [semacquire]:
strings.NewReader(...)
/usr/local/go/src/strings/reader.go:154
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc000b2f8f8, 0xc002d28000, 0xe9f, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:87 +0xf9
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d2a20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1338 [semacquire]:
strings.Replace(0xc004f02000, 0xfb9, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0xe, 0xc004f02000, 0xfb9)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0053a58f8, 0xc002d4e000, 0xfb8, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d2ae0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1339 [semacquire]:
strings.Replace(0xc0047be000, 0xa1a, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0x8, 0xc0047be000, 0xa1a)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0018df8f8, 0xc0044eca80, 0xa19, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d2ba0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1340 [semacquire]:
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:109 +0x42
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d2c60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1341 [semacquire]:
golang.org/x/net/html.NewTokenizerFragment(0xb7c280, 0xc004cdb340, 0x0, 0x0, 0x20)
/home/develop/go/src/golang.org/x/net/html/token.go:1213 +0x5d
golang.org/x/net/html.NewTokenizer(...)
/home/develop/go/src/golang.org/x/net/html/token.go:1201
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc004cdb340, 0x0, 0x0, 0x0, 0xc000777200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2337 +0x41
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc004cdb340, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0007778f8, 0xc001192580, 0xc0e, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d2d20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1342 [semacquire]:
strings.genSplit(0xc002385800, 0x7ef, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x0, 0xc005707340, 0xc0017f5360)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0017f53c0, 0xaa059b, 0x2, 0xc002385000, 0x7ef, 0xc002385000, 0xc0017f53b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002385000, 0x7ef, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0017f5518, 0xc005964f40, 0x4d, 0xc00277c050)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d2de0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1343 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc000a8c000, 0xa97, 0xc005481218, 0xc003be6380)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc000a8c000, 0xa97, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0054813c0, 0xaa04f9, 0x2, 0xc000a8c000, 0xa97, 0x5, 0x23d, 0xc00656ce60)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc000a8c000, 0xa97, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc005481518, 0xc00000e380, 0x48, 0xc00189a820)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d2ea0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1344 [semacquire]:
strings.Replace(0xc00149e600, 0x11d9, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0x10, 0xc00149e600, 0x11d9)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0053a98f8, 0xc00308e000, 0x11d7, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d2f60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1345 [semacquire]:
github.com/gigawattio/window.Rolling(0xc0064b3800, 0x17c, 0x17c, 0x3, 0x0, 0x17b, 0xc0064b3800)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0044a3518, 0xc001396900, 0xc001cfcb00)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d3020, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1346 [semacquire]:
strings.Replace(0xc0056be000, 0xe19, 0xaa042f, 0x2, 0xaa0af0, 0x3, 0x1, 0xc0006b42a0, 0xc0032a8700)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:175 +0x72
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc001f3d8f8, 0xc0056be000, 0xe19, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d30e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1347 [semacquire]:
bytes.makeSlice(0x1a96, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc004c10b10, 0x4, 0x1)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc004c10b10, 0xad0464, 0x4, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.render1(0xb84d20, 0xc004c10b10, 0xc002ba1180, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:163 +0x316
golang.org/x/net/html.render1(0xb84d20, 0xc004c10b10, 0xc002ba10a0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc004c10b10, 0xc000726230, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc004c10b10, 0xc000726150, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc004c10b10, 0xc0006f7a40, 0xb84d20, 0xc004c10b10)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc004c10b10, 0xc0006f7a40, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc0046801e0, 0x50d2a1, 0xc000c51840, 0x31, 0xaada21)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001f13518, 0xc00137fde0, 0xc005d7b620)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:285 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d31a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1348 [semacquire]:
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc0016a32e0, 0x432e00, 0xc0016a3210, 0x9a030e24, 0xc001ce0000)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:724 +0x716
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc0016a32e0, 0xc0001d5980, 0x5, 0xc0016a3230, 0x46620f)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc0016a32e0, 0xc0016a32f8, 0x43b3b7, 0x1252790, 0xc0016a3200, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xaa64ee, 0x5, 0x7fe2382f9a00, 0xc0016a3301, 0x10100000042ab7a, 0x0, 0x1077)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xaa64ee, 0x5, 0x0, 0x0, 0xaa0301)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc002f80ff0, 0xaa64ee, 0x5, 0x1a89)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).getTitleUnmodified(0xc0016a3518, 0xc002409d80, 0x1077, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:57 +0x47
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:128 +0x282
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d3260, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1349 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0022f5000, 0xe1b, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:87 +0x3aa
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0022f5000, 0xe1b, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0022f5000, 0xe1b, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0022f5000, 0xe1b, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0022f5000, 0xe1b, 0x0, 0x0, 0xe1b, 0xc0022f5000, 0xe1b)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0022f5000, 0xe1b, 0xe1b, 0x1000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc004396e80, 0x3f, 0xc0022f5000, 0xe1b, 0xc0022f5000, 0xe1b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc004396e80, 0x3f, 0xc0022f5000, 0xe1b, 0xaa0378, 0x1, 0xc0022f5000, 0xe1b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d3320, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1350 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc005097900, 0xc3f, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:71 +0x129
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc005097900, 0xc3f, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc005097900, 0xc3f, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc005097900, 0xc3f, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc005097900, 0xc3f, 0x0, 0x0, 0xc3f, 0xc005097900, 0xc3f)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc005097900, 0xc3f, 0xc3f, 0x1000)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0034566c0, 0x39, 0xc005097900, 0xc3f, 0xc005097900, 0xc3f, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0034566c0, 0x39, 0xc005097900, 0xc3f, 0xaa0378, 0x1, 0xc005097900, 0xc3f, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d33e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1351 [semacquire]:
strings.Replace(0xc0063a1000, 0xf35, 0xaa042f, 0x2, 0xaa0af0, 0x3, 0x2, 0xc03f80007f, 0x0)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:175 +0x72
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0044738f8, 0xc0063a1000, 0xf35, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d34a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1352 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc001708000, 0xab0, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc001708000, 0xab0, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc001708000, 0xab0, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc001708000, 0xab0, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc001708000, 0xab0, 0x0, 0x0, 0xab0, 0xc001708000, 0xab0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc001708000, 0xab0, 0xab0, 0xd80)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc004af9145, 0x30, 0xc001708000, 0xab0, 0xc001708000, 0xab0, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc004af9145, 0x30, 0xc001708000, 0xab0, 0xaa0378, 0x1, 0xc001708000, 0xab0, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d3560, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1353 [semacquire]:
strings.genSplit(0xc0034b2380, 0xc86, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc005076000, 0x29b, 0x29b)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0038593c0, 0xaa04d9, 0x2, 0xc006054d80, 0xc86, 0x1, 0x29b, 0xc004cdaac0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc006054d80, 0xc86, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003859518, 0xc0047e3920, 0x5d, 0xc001ffe840)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d3620, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1354 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc000643930, 0xc005bc2240, 0x23e)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc000643930, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc000643930)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc000643930, 0xc0003a8fc0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00115a720, 0x0, 0x0, 0x0, 0xc0016db200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00115a720, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0016db8f8, 0xc0048f4000, 0x1d97, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d36e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1355 [semacquire]:
github.com/PuerkitoBio/goquery.appendWithoutDuplicates(0xc0063b25c0, 0x8, 0x8, 0xc002f13f80, 0x10, 0x10, 0xc00432f020, 0xc00627fba0, 0x1, 0x8)
/home/develop/go/src/github.com/PuerkitoBio/goquery/utilities.go:137 +0x1cf
github.com/PuerkitoBio/goquery.mapNodes(0xc00627f690, 0x1, 0x1, 0xc00432f0f0, 0x7fe260fee9b8, 0x0, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:694 +0x159
github.com/PuerkitoBio/goquery.getSiblingNodes(0xc00627f690, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc00627fba0, ...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:598 +0x101
github.com/PuerkitoBio/goquery.(*Selection).Siblings(0xc002496ab0, 0xc00191abd0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:268 +0x59
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc00432f518, 0xc002496ab0, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:517 +0x20f
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00432f518, 0xc002227cc0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d37a0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1356 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc001bda270, 0xc0044b0000, 0x16b)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc001bda270, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc001bda270)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc001bda270, 0xc0003a9c00, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00115a7e0, 0x0, 0x0, 0x0, 0xc0018fd200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00115a7e0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0018fd8f8, 0xc003430000, 0x1402, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d3860, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1357 [semacquire]:
strings.Replace(0xc0004a8800, 0xb9c, 0xaa04bd, 0x2, 0xaa03a3, 0x1, 0x2, 0xb7b900, 0xc00005e060)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/jaytaylor/html2text.FromHTMLNode(0xc004f19030, 0x0, 0x0, 0x0, 0x0, 0xc004f19030, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:81 +0x158
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc0058106c0, 0x0, 0x0, 0x0, 0x1000, 0xc005813000, 0xf01, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc005813000, 0xf01, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc004397c00)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc0053e9518, 0xc00137e320, 0xc004909dc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d3920, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1358 [semacquire]:
strings.makeCutsetFunc(0xaa4e4b, 0x5, 0xa7c820)
/usr/local/go/src/strings/strings.go:830 +0x4a
strings.TrimLeft(0xc00403e6a8, 0x5, 0xaa4e4b, 0x5, 0x5, 0x0)
/usr/local/go/src/strings/strings.go:855 +0x61
golang.org/x/net/html.inHeadIM(0xc0052f2000, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:626 +0x969
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0052f2000)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0052f2000, 0xc0040d8070, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00229d8c0, 0x0, 0x0, 0x0, 0xc003c91200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00229d8c0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc003c918f8, 0xc00144d400, 0xab3, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d39e0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1359 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc001884d00, 0xc0038172c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0038173c0, 0xaa04fd, 0x2, 0xc00106db00, 0xc87, 0x1, 0x283, 0xc000c45960)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00106db00, 0xc87, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003817518, 0xc000c45560, 0x55, 0xc002256ba0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d3aa0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1360 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc001a4cd00, 0x887, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:43 +0xd5
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc001a4cd00, 0x887, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc001a4cd00, 0x887, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc001a4cd00, 0x887, 0x0, 0x0, 0x887, 0xc001a4cd00, 0x887)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc001a4cd00, 0x887, 0x887, 0x900)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0011277cd, 0x3e, 0xc001a4cd00, 0x887, 0xc001a4cd00, 0x887, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0011277cd, 0x3e, 0xc001a4cd00, 0x887, 0xaa0378, 0x1, 0xc001a4cd00, 0x887, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d3b60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1361 [semacquire]:
strings.NewReader(...)
/usr/local/go/src/strings/reader.go:154
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0018fb8f8, 0xc0049dc000, 0x1f15, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:87 +0xf9
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d3c20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1362 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc0035efe10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc0035efee0, 0xc0049b0ee0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0010b1440, 0x0, 0x0, 0x0, 0xc0029d5200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0010b1440, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0029d58f8, 0xc004bc9800, 0xb7e, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d3ce0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1363 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc005bcf4a0, 0xc0040212c0, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0040213c0, 0xaa058b, 0x2, 0xc002946000, 0x74a, 0xc002946000, 0xc0040213b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002946000, 0x74a, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc004021518, 0xc00115aa40, 0x30, 0xc0006e7380)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d3da0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1364 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc001bdbe10, 0xc0026dc1e0, 0x49)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc001bdbe10, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc001bdbe10)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc001bdbe10, 0xc0005d0380, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00115ad60, 0x0, 0x0, 0x0, 0xc001acf200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00115ad60, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc001acf8f8, 0xc00136d800, 0xbc3, 0xc001c1b4bc, 0x0, 0xc001c1b4bd)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d3e60, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1365 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc0029ed040, 0xc002f837a0, 0x90)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc0029ed040, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0029ed040)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0029ed040, 0xc000928ee0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc001c5e2e0, 0x0, 0x0, 0x0, 0xc000c31200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc001c5e2e0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc000c318f8, 0xc001eea200, 0x1aa4, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0001d3f20, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1366 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001525588, 0xaa04f9, 0x2, 0xc0046faf20, 0x152, 0x0, 0xc0015253d8, 0xc001246f28)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).isBoostable(0xc001525518, 0xc002d3dc80, 0x1)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:528 +0xfd
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc001525518, 0xc0000b34c0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:402 +0x2da
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312000, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1367 [semacquire]:
golang.org/x/net/html.(*parser).addText(0xc000387930, 0xc002509b80, 0x136)
/home/develop/go/src/golang.org/x/net/html/parse.go:318 +0x128
golang.org/x/net/html.inBodyIM(0xc000387930, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:861 +0x2c3
golang.org/x/net/html.(*parser).parseCurrentToken(0xc000387930)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc000387930, 0xc0000fdb90, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0000b28e0, 0x0, 0x0, 0x0, 0xc0045b3200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0000b28e0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0045b38f8, 0xc000608000, 0x10df, 0x3, 0x3, 0xc000ccbc80)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003120c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1368 [semacquire]:
bytes.makeSlice(0xb74, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00518b8c0, 0x40, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00518b8c0, 0xc0058fe000, 0x40, 0x6, 0xffffffffffffffff, 0xad0184)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc00518b8c0, 0xc0058fe000, 0x40, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/escape.go:229 +0x22f
golang.org/x/net/html.render1(0xb84d20, 0xc00518b8c0, 0xc00232e8c0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc00518b8c0, 0xc00232e5b0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00518b8c0, 0xc002a2e4d0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00518b8c0, 0xc003928930, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00518b8c0, 0xc00659f030, 0xb84d20, 0xc00518b8c0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc00518b8c0, 0xc00659f030, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc00518b4d0, 0x16fd3496a, 0x1251a20, 0xc0027eeb80, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312180, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1369 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000eb73c0, 0xaa059b, 0x2, 0xc002843500, 0x6f8, 0x15, 0x192, 0xc004909140)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc002843500, 0x6f8, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc000eb7518, 0xc0032a9540, 0x30, 0xc0008a0ab0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312240, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1370 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe19c455320, 0x7fe19c002120)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc00088ca00, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc00088c9f0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc0016b9c00, 0x36a, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc002ed8040, 0x3c, 0xc0016b9c00, 0x36a, 0xc0016b9c00, 0x36a, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc002ed8040, 0x3c, 0xc0016b9c00, 0x36a, 0xaa0378, 0x1, 0xc0016b9c00, 0x36a, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312300, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1371 [semacquire]:
regexp.(*Regexp).replaceAll(0xc000149400, 0x0, 0x0, 0x0, 0xc001e18120, 0x105, 0x2, 0xc0021e2198, 0xc00000000a, 0x2, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc001e18120, 0x105, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021e30c0, 0xc000459e30, 0xc0021e21b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0021e30c0, 0xc000459dc0, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc0021e30c0, 0xc000459dc0, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc0021e30c0, 0xc000459dc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc0021e30c0, 0xc000459dc0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0021e30c0, 0xc001b2f500, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc001f170c0, 0xc001b2f500, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001f170c0, 0xc001b2f500, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0021e30c0, 0xc001b2f420, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc001f170c0, 0xc001b2f420, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001f170c0, 0xc001b2f420, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0021e30c0, 0xc001b2ec40, 0xc001d7ae00, 0x7fe260fedc28)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc001f170c0, 0xc001b2ec40, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001f170c0, 0xc001b2ec40, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc0021e30c0, 0xc001b2eb60, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc001f170c0, 0xc001b2eb60, 0x54b170, 0xc002d2a680)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc001b2eb60, 0x0, 0x0, 0x0, 0x0, 0xc001b2eb60, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc001e3c3f0, 0x0, 0x0, 0x0, 0x1300, 0xc001f3e000, 0x1184, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc001f3e000, 0x1184, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc002340c00)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001f17518, 0xc004c38120, 0xc005b75400)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003123c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1372 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe2340a1800, 0x7fe1e021ff60)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc0035ca500, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc0035ca4f0, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc002bf8000, 0x87b, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc0046af0c0, 0x3e, 0xc002bf8000, 0x87b, 0xc002bf8000, 0x87b, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc0046af0c0, 0x3e, 0xc002bf8000, 0x87b, 0xaa0378, 0x1, 0xc002bf8000, 0x87b, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312480, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1373 [runnable]:
_/home/ec2-user/environment/cluster2/cluster2/fasttgo._Cfunc_NewHandle(0x7fe1ac00db80, 0x7fe1ac00ea00)
_cgo_gotypes.go:80 +0x4a
_/home/ec2-user/environment/cluster2/cluster2/fasttgo.Open(0xc003598b80, 0xd, 0x0)
/home/ec2-user/environment/cluster2/cluster2/fasttgo/fasttext.go:124 +0x83
_/home/ec2-user/environment/cluster2/cluster2/base.GetModelInstance(0xc003598b60, 0x7, 0x5)
/home/ec2-user/environment/cluster2/cluster2/base/model.go:19 +0x117
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PredictCategory(0xaa04f9, 0x2, 0xc001e0e480, 0x463, 0x572a20, 0xc0001fb400, 0xaa0378)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:28 +0x75
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc004750480, 0x38, 0xc001e0e480, 0x463, 0xc001e0e480, 0x463, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:19 +0xd9
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc004750480, 0x38, 0xc001e0e480, 0x463, 0xaa0378, 0x1, 0xc001e0e480, 0x463, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312540, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1374 [semacquire]:
strings.genSplit(0xc0014f7c00, 0x6d8, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0x785, 0xc0014f6e00, 0x6d8)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc001fdd588, 0xaa04f9, 0x2, 0xc0014f6e00, 0x6d8, 0x6, 0xf, 0xc002300940)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc001fdd518, 0xc001000760, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312600, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1376 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc002f540d0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:901 +0x820
golang.org/x/net/html.(*parser).parseCurrentToken(0xc002f540d0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc002f540d0, 0xc005526380, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00522cd20, 0x0, 0x0, 0x0, 0xc00451d200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00522cd20, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00451d8f8, 0xc004501500, 0x14d5, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312780, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1377 [semacquire]:
strings.Replace(0xc0047bf500, 0x9a0, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0xa, 0xc0047bf500, 0x9a0)
/usr/local/go/src/strings/strings.go:959 +0x466
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0023978f8, 0xc0037a8000, 0x99e, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312840, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1378 [runnable]:
strings.(*Builder).WriteByte(...)
/usr/local/go/src/strings/builder.go:98
strings.Map(0xae62d0, 0xc0044e2031, 0x559, 0x437c3e, 0xc00004d500)
/usr/local/go/src/strings/strings.go:508 +0x333
strings.ToLower(0xc0044e2000, 0x58a, 0x42f9b8, 0xc002f892e0)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc002f893c0, 0xaa04fd, 0x2, 0xc0044e2000, 0x58a, 0xc0044e2000, 0xc002f893b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0044e2000, 0x58a, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc002f89518, 0xc0027ee200, 0x22, 0xc000a14090)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312900, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1379 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00077d3c0, 0xaa04f9, 0x2, 0xc0005fac00, 0xa9c, 0xc0005fac00, 0xc00077d3b0, 0x7544a2)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0005fac00, 0xa9c, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00077d518, 0xc003213800, 0x4c, 0xc00002b360)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003129c0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1380 [semacquire]:
strings.(*Builder).grow(...)
/usr/local/go/src/strings/builder.go:68
strings.(*Builder).Grow(...)
/usr/local/go/src/strings/builder.go:82
strings.Map(0xae62d0, 0xc00555a000, 0x251f, 0xc0024db218, 0xc004303180)
/usr/local/go/src/strings/strings.go:485 +0x4b7
strings.ToLower(0xc00555a000, 0x251f, 0x1, 0x1)
/usr/local/go/src/strings/strings.go:610 +0x8f
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc0024db3c0, 0xaa04df, 0x2, 0xc00555a000, 0x251f, 0x1, 0x6cf, 0xc0004e7d40)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:75 +0x98
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc00555a000, 0x251f, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc0024db518, 0xc0004e7960, 0x47, 0xc00002b400)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312a80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1382 [semacquire]:
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc000f5d588, 0xaa04f9, 0x2, 0xc0046fa840, 0x160, 0x13, 0xc000f5d290, 0x42d308)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x261
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore.func1(0x3, 0xc001999e90)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:635 +0x82
github.com/PuerkitoBio/goquery.(*Selection).Each(0xc003daedb0, 0xc000f5d320, 0x1)
/home/develop/go/src/github.com/PuerkitoBio/goquery/iteration.go:10 +0x53
github.com/advancedlogic/GoOse.(*ContentExtractor).getSiblingsScore(0xc000f5d518, 0xc003daeb40, 0xc0027efdc0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:633 +0xa5
github.com/advancedlogic/GoOse.(*ContentExtractor).addSiblings(0xc000f5d518, 0xc003daeb40, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:697 +0x57
github.com/advancedlogic/GoOse.(*ContentExtractor).PostCleanup(0xc000f5d518, 0xc003daeb40, 0xc003daeb40)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:718 +0x3f
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:158 +0x788
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312c00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1383 [semacquire]:
github.com/jaytaylor/html2text.(*textifyTraverseContext).breakLongLines(0xc00041a070, 0xc005a5c000, 0x13, 0x2, 0x2, 0xa)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:462 +0x58c
github.com/jaytaylor/html2text.(*textifyTraverseContext).emit(0xc00041a070, 0xc005a5c000, 0x13, 0x13, 0x1)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:426 +0x63
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00041a070, 0xc001c1d2d0, 0xc0003ab600, 0xc000419fa0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:404 +0x5b
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00041a070, 0xc001c1d260, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00041a4a0, 0xc001c1d260, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:155 +0x7cc
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00041a4a0, 0xc001c1d260, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00041a4a0, 0xc001c1d110, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc00041b0c0, 0xc001c1d110, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:229 +0x376
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc00041b0c0, 0xc001c1d110, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00041b0c0, 0xc001b80d20, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002f8b0c0, 0xc001b80d20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002f8b0c0, 0xc001b80d20, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00041b0c0, 0xc001b80c40, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002f8b0c0, 0xc001b80c40, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002f8b0c0, 0xc001b80c40, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00041b0c0, 0xc00034c5b0, 0xc003be6a80, 0x7fe260fec108)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc002f8b0c0, 0xc00034c5b0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002f8b0c0, 0xc00034c5b0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc00041b0c0, 0xc00034c4d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc002f8b0c0, 0xc00034c4d0, 0x54b170, 0xc002eed6c0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc00034c4d0, 0x0, 0x0, 0x0, 0x0, 0xc00034c4d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc003daf920, 0x0, 0x0, 0x0, 0xc00, 0xc0012f9c00, 0xaff, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc0012f9c00, 0xaff, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc003355540)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002f8b518, 0xc000c44840, 0xc000c45160)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312cc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1384 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312d80, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:301 +0x215
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1385 [semacquire]:
strings.genSplit(0xc005934880, 0xcc3, 0xaa0378, 0x1, 0x0, 0xffffffffffffffff, 0xc001b6f000, 0x2e0, 0x2e0)
/usr/local/go/src/strings/strings.go:256 +0x68
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00335f3c0, 0xaa04f9, 0x2, 0xc0013de880, 0xcc3, 0x7, 0x2e0, 0xc005b74020)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:76 +0xd8
github.com/advancedlogic/GoOse.StopWords.SimpleLanguageDetector(0xc00038e060, 0xc0013de880, 0xcc3, 0x0, 0x127e7a0)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:101 +0xdc
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc00335f518, 0xc0027ee420, 0x3b, 0xc005c23a0f)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x146
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312e40, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1386 [semacquire]:
strings.Replace(0xc001de3200, 0xc49, 0xaa042f, 0x2, 0xaa0af0, 0x3, 0x1, 0xc03f800063, 0x0)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:175 +0x72
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0043118f8, 0xc001de3200, 0xc49, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312f00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1387 [runnable]:
regexp/syntax.(*compiler).inst(...)
/usr/local/go/src/regexp/syntax/compile.go:173
regexp/syntax.Compile(0xc0050bbb90, 0xc0050bbb90, 0x1, 0x1)
/usr/local/go/src/regexp/syntax/compile.go:84 +0x241
regexp.compile(0xaa5b70, 0x5, 0xc0020000d4, 0xc005f0f500, 0xc001bb1248, 0x509c73)
/usr/local/go/src/regexp/regexp.go:178 +0xd7
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
regexp.MustCompile(0xaa5b70, 0x5, 0xaa0378)
/usr/local/go/src/regexp/regexp.go:309 +0x4b
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc002f8b518, 0xc0024082a0, 0xc002408da0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:324 +0x7f8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000312fc0, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1388 [semacquire]:
github.com/gigawattio/window.Rolling(0xc003a19000, 0xf9, 0xf9, 0x3, 0x0, 0xf8, 0xc003a19000)
/home/develop/go/src/github.com/gigawattio/window/rolling.go:15 +0x140
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc001721518, 0xc002408e80, 0xc0024097e0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:329 +0x8ed
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000313080, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1389 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc002eec000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc002eec0d0, 0xc005274a10, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0027ee100, 0x0, 0x0, 0x0, 0xc002b97200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0027ee100, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc002b978f8, 0xc003090600, 0x116b, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000313140, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1390 [runnable]:
strings.genSplit(0xc003749643, 0x772, 0xaa0378, 0x1, 0x0, 0x22b, 0x1, 0xc000480000, 0x22b)
/usr/local/go/src/strings/strings.go:264 +0x133
strings.Split(...)
/usr/local/go/src/strings/strings.go:308
github.com/advancedlogic/GoOse.(*ContentExtractor).isHighLinkDensity(0xc0029e1518, 0xc004c08660, 0x2)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:567 +0xd5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc0029e1518, 0xc00608d800, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:386 +0x1ae
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000313200, 0x59)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1391 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.count(0xc0041d2500, 0x4c9, 0xae4d68)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:85 +0x317
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.getTrigramsWithPositions(0xc0041d2500, 0x4c9, 0xae4d20)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languagesTrigrams.go:39 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangInProfiles(0xc0041d2500, 0x4c9, 0x0, 0x0, 0xc000234840, 0x115f3c0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:135 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.detectLangBaseOnScript(0xc0041d2500, 0x4c9, 0x0, 0x0, 0x115f500, 0x46e6ce, 0x0)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:79 +0x582
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectWithOptions(0xc0041d2500, 0x4c9, 0x0, 0x0, 0x4c9, 0xc0041d2500, 0x4c9)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:60 +0x79
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.Detect(...)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:43
_/home/ec2-user/environment/cluster2/cluster2/languagesDetect.DetectLang(0xc0041d2500, 0x4c9, 0x4c9, 0x500)
/home/ec2-user/environment/cluster2/cluster2/languagesDetect/languages.go:48 +0x3d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.DetectCategories(0xc004af9090, 0x41, 0xc0041d2500, 0x4c9, 0xc0041d2500, 0x4c9, 0xc0000ace10, 0x0, 0x0)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/categories.go:11 +0x3c
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.PrepareThreads(0xc004af9090, 0x41, 0xc0041d2500, 0x4c9, 0xaa0378, 0x1, 0xc0041d2500, 0x4c9, 0x11, 0xacd3bb)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/threads.go:13 +0x4d
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003132c0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:307 +0x3ae
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1392 [semacquire]:
github.com/fatih/set.(*Set).Add(0xc001cfc560, 0xc00085d220, 0x1, 0x1)
/home/develop/go/src/github.com/fatih/set/set_ts.go:35 +0xcd
github.com/advancedlogic/GoOse.(*StopWords).stopWordsCount(0xc00085d588, 0xaa04f9, 0x2, 0xc0066a6000, 0x12da, 0x7, 0xf, 0xc004909740)
/home/develop/go/src/github.com/advancedlogic/GoOse/stopwords.go:82 +0x2b5
github.com/advancedlogic/GoOse.(*ContentExtractor).CalculateBestNode(0xc00085d518, 0xc0047e24c0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:424 +0x3c6
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:156 +0x6a1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000313380, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1393 [semacquire]:
regexp.newQueue(...)
/usr/local/go/src/regexp/onepass.go:140
regexp.makeOnePass(0xc0032c66c0, 0xc0032c66c0)
/usr/local/go/src/regexp/onepass.go:307 +0x148
regexp.compileOnePass(0xc0032c6600, 0xc0032c6600)
/usr/local/go/src/regexp/onepass.go:500 +0x129
regexp.compile(0xabc10c, 0x11, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc005415300, 0x0, 0xabc106, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc005415300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc005415300, 0x0, 0xb856e0, 0xc00039e248, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc005415300, 0xc002b0bb20, 0x0, 0x8, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc005415300, 0x42ca96, 0xc0032c65a0, 0x30, 0x28, 0xa4b3a0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xabc101, 0x1d, 0xc005415370, 0x42d308, 0x30, 0xa4b3a0, 0xb85601)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xabc101, 0x1d, 0xc00522cb40, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc001d09f50, 0xabc101, 0x1d, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc005415518, 0xc001dd3580, 0xabc101, 0x1d, 0x4b, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:133 +0x3a7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000313440, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1394 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.inBodyIM(0xc000643e10, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:901 +0x820
golang.org/x/net/html.(*parser).parseCurrentToken(0xc000643e10)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc000643e10, 0xc0003a99d0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00115a7c0, 0x0, 0x0, 0x0, 0xc0016d7200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00115a7c0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0016d78f8, 0xc000d65300, 0x1274, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000313500, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1395 [semacquire]:
strings.Replace(0xc001f53300, 0x11c5, 0xaa36c2, 0x4, 0xaa5832, 0x5, 0x11, 0xc001f53300, 0x11c5)
/usr/local/go/src/strings/strings.go:941 +0x109
github.com/advancedlogic/GoOse.Crawler.addSpacesBetweenTags(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:179 +0x19e
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00387f8f8, 0xc004bb5900, 0x11c2, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:85 +0xc8
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003135c0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1396 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000313680, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x27d
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1397 [semacquire]:
bytes.makeSlice(0x5b7, 0x0, 0x0, 0x0)
/usr/local/go/src/bytes/buffer.go:229 +0x73
bytes.(*Buffer).grow(0xc00118aa50, 0x15, 0x0)
/usr/local/go/src/bytes/buffer.go:142 +0x15b
bytes.(*Buffer).WriteString(0xc00118aa50, 0xc004397dc0, 0x15, 0x6, 0x15, 0x0)
/usr/local/go/src/bytes/buffer.go:184 +0xdc
golang.org/x/net/html.escape(0xb84d20, 0xc00118aa50, 0xc004397dc0, 0x39, 0x45624c, 0xc004022c90)
/home/develop/go/src/golang.org/x/net/html/escape.go:201 +0xe5
golang.org/x/net/html.render1(0xb84d20, 0xc00118aa50, 0xc0003fc3f0, 0x0, 0x127e7a0)
/home/develop/go/src/golang.org/x/net/html/render.go:74 +0x10d
golang.org/x/net/html.render1(0xb84d20, 0xc00118aa50, 0xc0003fc380, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00118aa50, 0xc0003fc2a0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00118aa50, 0xc0003fc1c0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render1(0xb84d20, 0xc00118aa50, 0xc0003fbab0, 0xb84d20, 0xc00118aa50)
/home/develop/go/src/golang.org/x/net/html/render.go:218 +0x8c1
golang.org/x/net/html.render(...)
/home/develop/go/src/golang.org/x/net/html/render.go:61
golang.org/x/net/html.Render(0xb7b5e0, 0xc00118aa50, 0xc0003fbab0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/render.go:47 +0x22e
github.com/PuerkitoBio/goquery.(*Selection).Html(0xc003232a20, 0x171fec5e9, 0x1251a20, 0xc00137e520, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:104 +0x90
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:121 +0x1b7
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000313740, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1398 [semacquire]:
regexp.makeOnePass(0xc005186ae0, 0xc005186ae0)
/usr/local/go/src/regexp/onepass.go:309 +0x20e
regexp.compileOnePass(0xc005186a80, 0xc005186a80)
/usr/local/go/src/regexp/onepass.go:500 +0x129
regexp.compile(0xaba6cc, 0xe, 0xd4, 0x10, 0xa2acc0, 0x4)
/usr/local/go/src/regexp/regexp.go:189 +0x106
regexp.Compile(...)
/usr/local/go/src/regexp/regexp.go:133
github.com/andybalholm/cascadia.(*parser).parseRegex(0xc001ad3300, 0x0, 0xaba6c6, 0x4)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:246 +0xc2
github.com/andybalholm/cascadia.(*parser).parseAttributeSelector(0xc001ad3300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:395 +0x6ef
github.com/andybalholm/cascadia.(*parser).parseSimpleSelectorSequence(0xc001ad3300, 0x0, 0x0, 0x0, 0xb856e0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:741 +0x616
github.com/andybalholm/cascadia.(*parser).parseSelector(0xc001ad3300, 0x75a856, 0x0, 0xc0003fc690, 0x0)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:779 +0x3d
github.com/andybalholm/cascadia.(*parser).parseSelectorGroup(0xc001ad3300, 0xc001ad32c8, 0xc001ad3350, 0x42ca96, 0xc005186a50, 0x30)
/home/develop/go/src/github.com/andybalholm/cascadia/parser.go:820 +0x32
github.com/andybalholm/cascadia.ParseGroup(0xaba6c1, 0x1a, 0x1, 0xc001ad3368, 0xc001ad3380, 0x42d308, 0x30)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:67 +0x77
github.com/andybalholm/cascadia.Compile(0xaba6c1, 0x1a, 0x1, 0xb856e0, 0xc004908b00)
/home/develop/go/src/github.com/andybalholm/cascadia/selector.go:104 +0x39
github.com/PuerkitoBio/goquery.compileMatcher(...)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:129
github.com/PuerkitoBio/goquery.(*Selection).Find(0xc003bb7890, 0xaba6c1, 0x1a, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/traversal.go:24 +0x39
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaContentWithSelector(0xc001ad3518, 0xc005c54160, 0xaba6c1, 0x1a, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:174 +0x46
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:134 +0x401
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000313800, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1399 [semacquire]:
regexp.(*Regexp).ReplaceAllString(0xc000149400, 0xc0049065a0, 0x10b, 0xaa0378, 0x1, 0x0, 0x0)
/usr/local/go/src/regexp/regexp.go:560 +0x135
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000ee90c0, 0xc0051a31f0, 0xc000ee81b0, 0xaa04bf)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:402 +0xb0
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc000ee90c0, 0xc0051a3180, 0x2, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).paragraphHandler(0xc000ee90c0, 0xc0051a3180, 0xc000000001, 0x8)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:295 +0x6b
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000ee90c0, 0xc0051a3180, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:265 +0x4f6
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000ee90c0, 0xc0051a3180, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc000ee90c0, 0xc002c872d0, 0xc000000001, 0x5)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000ee90c0, 0xc002c872d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000ee90c0, 0xc002c872d0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc000ee90c0, 0xc002c871f0, 0xc000000001, 0x3)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000ee90c0, 0xc002c871f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000ee90c0, 0xc002c871f0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc000ee90c0, 0xc000bfbb90, 0xc003be6380, 0xb6c046)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).handleElement(0xc000ee90c0, 0xc000bfbb90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:286 +0x4aa
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000ee90c0, 0xc000bfbb90, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:407 +0xf4
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverseChildren(0xc000ee90c0, 0xc000bfbab0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:413 +0x4e
github.com/jaytaylor/html2text.(*textifyTraverseContext).traverse(0xc000ee90c0, 0xc000bfbab0, 0x54b170, 0xc003d5cea0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:395 +0x125
github.com/jaytaylor/html2text.FromHTMLNode(0xc000bfbab0, 0x0, 0x0, 0x0, 0x0, 0xc000bfbab0, 0x0, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:76 +0xab
github.com/jaytaylor/html2text.FromReader(0xb7b620, 0xc00419e270, 0x0, 0x0, 0x0, 0x1000, 0xc005aec000, 0xeeb, 0x0)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:97 +0xd3
github.com/jaytaylor/html2text.FromString(0xc005aec000, 0xeeb, 0x0, 0x0, 0x0, 0x8, 0xffffffffffffffff, 0x4, 0xc00600ae80)
/home/develop/go/src/github.com/jaytaylor/html2text/html2text.go:103 +0x128
github.com/advancedlogic/GoOse.(*ContentExtractor).GetPublishDate(0xc000ee9518, 0xc005bce660, 0xc005bcf100)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:291 +0xf3
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:143 +0x99d
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc0003138c0, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1400 [semacquire]:
bytes.(*Buffer).String(...)
/usr/local/go/src/bytes/buffer.go:65
github.com/PuerkitoBio/goquery.(*Selection).Text(0xc003233560, 0xc000021da0, 0x0)
/home/develop/go/src/github.com/PuerkitoBio/goquery/property.go:82 +0x122
github.com/advancedlogic/GoOse.(*ContentExtractor).GetMetaLanguage(0xc003855518, 0xc00137e700, 0x39, 0xc004397f8e)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:148 +0x134
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:130 +0x308
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000313980, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1401 [semacquire]:
regexp.(*Regexp).replaceAll(0xc0001fb220, 0x0, 0x0, 0x0, 0xc001e1e000, 0x2af1, 0x2, 0xc005aaf218, 0xc002e2c0e8, 0x7de1b0, ...)
/usr/local/go/src/regexp/regexp.go:608 +0x573
regexp.(*Regexp).ReplaceAllString(0xc0001fb220, 0xc001e1e000, 0x2af1, 0xaa0378, 0x1, 0xc0024084a0, 0xc002e2c2d8)
/usr/local/go/src/regexp/regexp.go:557 +0x10a
github.com/advancedlogic/GoOse.(*outputFormatter).getOutputText(0xc005aaf388, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:121 +0x81
github.com/advancedlogic/GoOse.(*outputFormatter).getFormattedText(0xc005aaf388, 0xc005bb5680, 0xaa04f9, 0x2, 0x42d308, 0x8, 0x9ecc80, 0xc005aaf301, 0xc002e2c218)
/home/develop/go/src/github.com/advancedlogic/GoOse/outputformatter.go:45 +0x103
github.com/advancedlogic/GoOse.(*ContentExtractor).GetCleanTextAndLinks(0xc005aaf518, 0xc005bb5680, 0xaa04f9, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/extractor.go:365 +0xaf
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:160 +0x7d1
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000313a40, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1402 [semacquire]:
golang.org/x/net/html.(*Tokenizer).readStartTag(0xc001bdad00, 0x69)
/home/develop/go/src/golang.org/x/net/html/token.go:803 +0x100
golang.org/x/net/html.(*Tokenizer).Next(0xc001bdad00, 0x1)
/home/develop/go/src/golang.org/x/net/html/token.go:1021 +0x28c
golang.org/x/net/html.(*parser).parse(0xc001bdadd0, 0xc0004d79d0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2284 +0xad
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00115aa00, 0x0, 0x0, 0x0, 0xc00401f200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00115aa00, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc00401f8f8, 0xc003f99000, 0x17ae, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000313b00, 0x5a)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 1403 [semacquire]:
golang.org/x/net/html.(*Tokenizer).Token(0xc002eec1a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/home/develop/go/src/golang.org/x/net/html/token.go:1175 +0x3a8
golang.org/x/net/html.(*parser).parse(0xc002eec270, 0xc0052750a0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2285 +0xc1
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc0027ee140, 0x0, 0x0, 0x0, 0xc0021f1200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc0027ee140, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0021f18f8, 0xc002de4000, 0x1f2d, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000313bc0, 0x57)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 399 [semacquire]:
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000381860, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:301 +0x215
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a
goroutine 400 [semacquire]:
golang.org/x/net/html.(*parser).addElement(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:326
golang.org/x/net/html.(*parser).addFormattingElement(0xc0064d7930)
/home/develop/go/src/golang.org/x/net/html/parse.go:337 +0x67
golang.org/x/net/html.inBodyIM(0xc0064d7930, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:978 +0x4c5
golang.org/x/net/html.(*parser).parseCurrentToken(0xc0064d7930)
/home/develop/go/src/golang.org/x/net/html/parse.go:2266 +0x80
golang.org/x/net/html.(*parser).parse(0xc0064d7930, 0xc0066165b0, 0x0)
/home/develop/go/src/golang.org/x/net/html/parse.go:2292 +0x48
golang.org/x/net/html.ParseWithOptions(0xb7c280, 0xc00656c2a0, 0x0, 0x0, 0x0, 0xc0043cd200, 0x42d308, 0x20)
/home/develop/go/src/golang.org/x/net/html/parse.go:2350 +0x10b
golang.org/x/net/html.Parse(...)
/home/develop/go/src/golang.org/x/net/html/parse.go:2309
github.com/PuerkitoBio/goquery.NewDocumentFromReader(0xb7c280, 0xc00656c2a0, 0x1194, 0xaa04f9, 0x2)
/home/develop/go/src/github.com/PuerkitoBio/goquery/type.go:53 +0x4a
github.com/advancedlogic/GoOse.(*Crawler).Preprocess(0xc0043cd8f8, 0xc0034a6c00, 0x10d6, 0x0, 0x0, 0x0)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:88 +0x147
github.com/advancedlogic/GoOse.Crawler.Crawl(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/crawler.go:111 +0x7a
github.com/advancedlogic/GoOse.Goose.ExtractFromRawHTML(0xaa0378, 0x0, 0x1194, 0xaa04f9, 0x2, 0xab47fe, 0x10, 0xab52f4, 0x11, 0xacd3bb, ...)
/home/develop/go/src/github.com/advancedlogic/GoOse/goose.go:33 +0x142
_/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1.1(0xc000029c70, 0xc000010670, 0xc000010678, 0xc000381b00, 0x5b)
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:302 +0x2e8
created by _/home/ec2-user/environment/cluster2/cluster2/newsDetect.ThreadsCommand.func1
/home/ec2-user/environment/cluster2/cluster2/newsDetect/newsCommand.go:282 +0x18a