diff --git a/README.md b/README.md index 9695d1e..74675e3 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,10 @@ Gophie is a tool to help you search, stream and download movies from movie sites - AnimeOut - Takanimelist +### Korean + +- KDramaHood + Gophie also has [mobile](https://github.com/Go-phie/gophie-mobile) and [web](https://github.com/Go-phie/gophie-web) clients. ## Installation diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go new file mode 100644 index 0000000..8acb2d8 --- /dev/null +++ b/cmd/cmd_test.go @@ -0,0 +1,27 @@ +package cmd + +import ( + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestCmd(t *testing.T) { + Convey("CLI [Test listing engines]\n", t, func() { + args := []string{"engines", "list"} + rootCmd.SetArgs(args) + So(rootCmd.Execute(), ShouldBeNil) + }) + + Convey("CLI [Test current version]\n", t, func() { + args := []string{"version"} + rootCmd.SetArgs(args) + So(rootCmd.Execute(), ShouldBeNil) + }) + + Convey("CLI [Test clearing cache]\n", t, func() { + args := []string{"clear-cache"} + rootCmd.SetArgs(args) + So(rootCmd.Execute(), ShouldBeNil) + }) + +} diff --git a/cmd/list.go b/cmd/list.go index d98d4c6..762e240 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -61,7 +61,7 @@ func listPager(pageNum int) { index++ } searchResult := engine.SearchResult{ - Query: selectedMovie.Title + "EPISODES", + Query: selectedMovie.Title + " EPISODES", Movies: movieArray, } selectedMovie = processList(pageNum, selectedEngine, searchResult) diff --git a/cmd/search.go b/cmd/search.go index 50be638..13b1e88 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -83,7 +83,7 @@ func searchPager(param ...string) { index++ } searchResult := engine.SearchResult{ - Query: selectedMovie.Title + "EPISODES", + Query: selectedMovie.Title + " EPISODES", Movies: movieArray, } selectedMovie = processSearch(selectedEngine, searchResult, param...) diff --git a/cmd/version_num.go b/cmd/version_num.go new file mode 100644 index 0000000..2784c0d --- /dev/null +++ b/cmd/version_num.go @@ -0,0 +1,3 @@ +package cmd + +const Version = "0.3.6" diff --git a/engine/engine_test.go b/engine/engine_test.go index 57e087a..c9be1f6 100644 --- a/engine/engine_test.go +++ b/engine/engine_test.go @@ -4,7 +4,6 @@ import ( "fmt" "strings" "testing" - // _ "github.com/go-phie/gophie/cmd" ) func testResults(t *testing.T, engine Engine) { @@ -12,15 +11,17 @@ func testResults(t *testing.T, engine Engine) { var result SearchResult var searchTerm string fmt.Println(engine.String()) - if !strings.HasPrefix(engine.String(), "TvSeries") { - if strings.HasPrefix(engine.String(), "Anime") || strings.HasPrefix(engine.String(), "Takanime") { - searchTerm = "attack on titan" - } else { - searchTerm = "jumanji" - } - } else { - // search for the flash for movie series + // different search terms on engines + switch { + case strings.HasPrefix(engine.String(), "TvSeries"): searchTerm = "devs" + case strings.HasPrefix(engine.String(), "TakanimeList"), + strings.HasPrefix(engine.String(), "AnimeOut"): + searchTerm = "attack on titans" + case strings.HasPrefix(engine.String(), "KDramaHood"): + searchTerm = "flower of evil" + default: + searchTerm = "jumanji" } result = engine.Search(searchTerm) @@ -37,7 +38,7 @@ func testResults(t *testing.T, engine Engine) { } if movie.IsSeries == false { downloadlink := strings.ToLower(movie.DownloadLink.String()) - if !(strings.HasSuffix(downloadlink, "1") || strings.HasSuffix(downloadlink, ".mp4") || strings.Contains(downloadlink, ".mkv") || strings.Contains(downloadlink, ".avi") || strings.Contains(downloadlink, ".webm") || strings.Contains(downloadlink, "freeload") || strings.Contains(downloadlink, "download_token=") || strings.Contains(downloadlink, "mycoolmoviez") || strings.Contains(downloadlink, "server")) { + if !(strings.HasSuffix(downloadlink, "1") || strings.HasSuffix(downloadlink, ".mp4") || strings.Contains(downloadlink, ".mkv") || strings.Contains(downloadlink, ".avi") || strings.Contains(downloadlink, ".webm") || strings.Contains(downloadlink, "freeload") || strings.Contains(downloadlink, "download_token=") || strings.Contains(downloadlink, "mycoolmoviez") || strings.Contains(downloadlink, "server") || strings.Contains(downloadlink, "kdramahood")) { t.Errorf("Could not obtain link for single movie, linked returned is %v", downloadlink) } } diff --git a/engine/engines.go b/engine/engines.go index 61937c1..6695d21 100644 --- a/engine/engines.go +++ b/engine/engines.go @@ -100,6 +100,10 @@ func Scrape(engine Engine) ([]Movie, error) { log.Fatal(err) } + // c.OnHTML("div", func(e *colly.HTMLElement) { + // log.Debugf("%#v", e) + // }) + c.OnHTML(main, func(e *colly.HTMLElement) { e.ForEach(article, func(_ int, el *colly.HTMLElement) { movie, err := engine.parseSingleMovie(el, movieIndex) @@ -167,7 +171,9 @@ type Movie struct { Category string // csv of categories Cast string // csv of actors in movie UploadDate string - Source string // The Engine From which it is gotten from + Source string // The Engine From which it is gotten from + SubtitleLink *url.URL // single subtitle link + SubtitleLinks map[string]*url.URL // Subtitle links for a series } // MovieJSON : JSON structure of all downloadable movies @@ -175,6 +181,7 @@ type MovieJSON struct { Movie DownloadLink string SDownloadLink map[string]string + SubtitleLinks map[string]string } func (m *Movie) String() string { @@ -187,11 +194,16 @@ func (m *Movie) MarshalJSON() ([]byte, error) { for key, val := range m.SDownloadLink { sDownloadLink[key] = val.String() } + subtitleLinks := make(map[string]string) + for key, val := range m.SubtitleLinks { + subtitleLinks[key] = val.String() + } movie := MovieJSON{ Movie: *m, DownloadLink: m.DownloadLink.String(), SDownloadLink: sDownloadLink, + SubtitleLinks: subtitleLinks, } return json.Marshal(movie) @@ -244,6 +256,7 @@ func GetEngines() map[string]Engine { engines["coolmoviez"] = NewCoolMoviezEngine() engines["animeout"] = NewAnimeOutEngine() engines["takanimelist"] = NewTakanimeListEngine() + engines["kdramahood"] = NewKDramaHoodEngine() return engines } diff --git a/engine/kdramahood.go b/engine/kdramahood.go new file mode 100644 index 0000000..148bd41 --- /dev/null +++ b/engine/kdramahood.go @@ -0,0 +1,165 @@ +package engine + +import ( + "fmt" + "net/url" + "path" + "strconv" + "strings" + + "github.com/gocolly/colly/v2" + log "github.com/sirupsen/logrus" +) + +// KDramaHood : An Engine for KDramaHood +type KDramaHood struct { + Props +} + +// NewKDramaHoodEngine : create a new engine for scraping latest korean drama +func NewKDramaHoodEngine() *KDramaHood { + base := "https://kdramahood.com" + baseURL, err := url.Parse(base) + if err != nil { + log.Fatal(err) + } + // Search URL + searchURL, err := url.Parse(base) + if err != nil { + log.Fatal(err) + } + searchURL.Path = "/" + + // List URL + listURL, err := url.Parse(base) + if err != nil { + log.Fatal(err) + } + listURL.Path = "/home2/" + + dramaFeverEngine := KDramaHood{} + dramaFeverEngine.Name = "KDramaHood" + dramaFeverEngine.BaseURL = baseURL + dramaFeverEngine.Description = `Watch your favourite korean movie all in one place` + dramaFeverEngine.SearchURL = searchURL + dramaFeverEngine.ListURL = listURL + return &dramaFeverEngine +} + +// Engine Interface Methods + +func (engine *KDramaHood) String() string { + st := fmt.Sprintf("%s (%s)", engine.Name, engine.BaseURL) + return st +} + +func (engine *KDramaHood) getParseAttrs() (string, string, error) { + return "div.items", "div.item", nil +} + +func (engine *KDramaHood) parseSingleMovie(el *colly.HTMLElement, index int) (Movie, error) { + movie := Movie{ + Index: index, + IsSeries: true, + Source: engine.Name, + Size: "---MB", + } + switch engine.mode { + case SearchMode: + movie.Title = strings.TrimSpace(el.ChildText("span.tt")) + movie.Description = strings.TrimSpace(el.ChildText("span.ttx")) + case ListMode: + movie.Title = strings.TrimSpace(el.ChildAttr("img", "alt")) + movie.Description = strings.TrimSpace(el.ChildText("div.contenido")) + } + movie.CoverPhotoLink = el.ChildAttr("img", "src") + link := el.Request.AbsoluteURL(el.ChildAttr("a", "href")) + downloadLink, err := url.Parse(link) + + if err != nil { + log.Fatal(err) + } + movie.DownloadLink = downloadLink + movie.Category = "kdrama" + return movie, nil +} + +func (engine *KDramaHood) updateDownloadProps(downloadCollector *colly.Collector, movies *[]Movie) { + innerCollector := downloadCollector.Clone() + episodeMap := map[string]*url.URL{} + subtitleMap := map[string]*url.URL{} + downloadCollector.OnHTML("ul.episodios", func(e *colly.HTMLElement) { + // clear existing map + for k := range episodeMap { + delete(episodeMap, k) + } + for k := range subtitleMap { + delete(subtitleMap, k) + } + // create local targets + targetepisode := make(map[string]*url.URL) + targetsub := make(map[string]*url.URL) + movie := &(*movies)[getMovieIndexFromCtx(e.Request)] + e.ForEach("li", func(_ int, inn *colly.HTMLElement) { + innerCollector.Visit(inn.ChildAttr("a", "href")) + }) + + // deepcopy to localtargets + for k, v := range episodeMap { + targetepisode[k] = v + } + for k, v := range subtitleMap { + targetsub[k] = v + } + + movie.SDownloadLink = targetepisode + movie.SubtitleLinks = targetsub + }) + + innerCollector.OnHTML("div.linkstv", func(e *colly.HTMLElement) { + name := e.ChildAttr("a", "download") + links := e.ChildAttrs("a", "href") + if len(links) > 1 { + // select first link + movieLink, _ := url.Parse(links[0]) + // subtitle is always the last link + subLink, _ := url.Parse(links[len(links)-1]) + episodeMap[name] = movieLink + subtitleMap[name] = subLink + } + }) +} + +// List : list all the movies on a page +func (engine *KDramaHood) List(page int) SearchResult { + engine.mode = ListMode + result := SearchResult{ + Query: "List of Recent Uploads - Page " + strconv.Itoa(page), + } + pageParam := fmt.Sprintf("page/%v", strconv.Itoa(page)) + engine.ListURL.Path = path.Join(engine.ListURL.Path, pageParam) + movies, err := Scrape(engine) + if err != nil { + log.Fatal(err) + } + result.Movies = movies + return result +} + +// Search : Searches fzmovies for a particular query and return an array of movies +func (engine *KDramaHood) Search(param ...string) SearchResult { + query := param[0] + engine.mode = SearchMode + result := SearchResult{ + Query: query, + } + q := engine.SearchURL.Query() + q.Set("s", query) + engine.SearchURL.RawQuery = q.Encode() + movies, err := Scrape(engine) + if err != nil { + log.Fatal(err) + } + result.Movies = movies + return result +} diff --git a/go.mod b/go.mod index 8dbfcf6..e1abe9e 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/manifoldco/promptui v0.7.0 github.com/mitchellh/go-homedir v1.1.0 github.com/sirupsen/logrus v1.6.0 + github.com/smartystreets/goconvey v1.6.4 github.com/spf13/cobra v1.0.0 github.com/spf13/viper v1.7.0 github.com/tebeka/selenium v0.9.9 diff --git a/go.sum b/go.sum index 5a98ade..e56ba04 100644 --- a/go.sum +++ b/go.sum @@ -170,6 +170,7 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -207,6 +208,7 @@ github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6Pyu github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a h1:FaWFmfWdAUKbSCtOU2QjDaorUexogfaMgbipgYATUMU= github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= @@ -319,7 +321,9 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -493,6 +497,7 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/reference/Gophie.v1.yaml b/reference/Gophie.v1.yaml index 5151f00..8a0927e 100644 --- a/reference/Gophie.v1.yaml +++ b/reference/Gophie.v1.yaml @@ -39,6 +39,10 @@ info: - [AnimeOut](https://www.animeout.xyz) - [TakanimeList] (https://takanimelist.best) + + ### Korean + + - [KDramaHood](https://kdramahood.com) servers: - url: 'https://deploy-gophie.herokuapp.com' description: Heroku server