# HG changeset patch # User anatofuz # Date 1585635136 -32400 # Node ID fca852bfd5007bd4678a41f52ea12b4aae873075 # Parent a0d23f38344d0c9536b9efa188ac528e48ecfb0b ... diff -r a0d23f38344d -r fca852bfd500 cmd_donwload.go --- a/cmd_donwload.go Tue Mar 31 13:40:06 2020 +0900 +++ b/cmd_donwload.go Tue Mar 31 15:12:16 2020 +0900 @@ -27,6 +27,14 @@ } arr3 := []string{"601495001", "600625001"} - err = dh.LecIDStoDonwlodSyllabus(ctx, arr3) + lwps, err := dh.LecIDStoDonwlodSyllabus(ctx, arr3) + fmt.Println(lwps) + for _, lwp := range *lwps { + lec, err := dh.LecIDwFilePath2LectureStruct(&lwp) + if err != nil { + return err + } + fmt.Println(lec) + } return err } diff -r a0d23f38344d -r fca852bfd500 syllabus/getSyllabus.go --- a/syllabus/getSyllabus.go Tue Mar 31 13:40:06 2020 +0900 +++ b/syllabus/getSyllabus.go Tue Mar 31 15:12:16 2020 +0900 @@ -3,6 +3,7 @@ import ( "bufio" "context" + "fmt" "io" "net/http" "net/url" @@ -11,27 +12,11 @@ "path/filepath" "strconv" "strings" - "sync" "time" "github.com/pkg/errors" ) -const ( - monday int = iota - tuesday - wednesday - thursday - friday -) - -//GetSyllabus is use main function struct. members using download html operation -type GetSyllabus struct { - year int - term string - outputdir string -} - // LectureDay include day of week (0~4, error -> 5), period, lastpriod (1~6) type LectureDay struct { DayOfWeek int @@ -43,10 +28,15 @@ type Lecture struct { ID string Name string - Day *LectureDay + Day LectureDay Teacher string } +type LectureWPath struct { + ID string + Path string +} + //CreateGetSyllabus is constructor and initialize from now time func CreateGetSyllabus() *GetSyllabus { var gs GetSyllabus @@ -85,66 +75,81 @@ return true, nil } -func (g *GetSyllabus) LecIDStoDonwlodSyllabus(ctx context.Context, lectureIDs []string) error { - var wg sync.WaitGroup +func (g *GetSyllabus) LecIDStoDonwlodSyllabus(ctx context.Context, lectureIDs []string) (*[]LectureWPath, error) { + //var wg sync.WaitGroup + ch := make(chan LectureWPath, len(lectureIDs)) for _, id := range lectureIDs { - wg.Add(1) + //wg.Add(1) go func(id string) { - defer wg.Done() - g.LecIDtoDownloadSyllabus(id) + //defer wg.Done() + outputPath, _ := g.LecIDtoDownloadSyllabus(id) + ch <- LectureWPath{ + ID: id, + Path: outputPath, + } }(id) } - wg.Wait() - return nil + //wg.Wait() + + var lwps []LectureWPath + for range lectureIDs { + lwps = append(lwps, <-ch) + } + return &lwps, nil } //LecIDtoDownloadSyllabus is download from lecture ID -func (g *GetSyllabus) LecIDtoDownloadSyllabus(lectureID string) error { +func (g *GetSyllabus) LecIDtoDownloadSyllabus(lectureID string) (string, error) { var strBuilder strings.Builder strBuilder.WriteString(lectureID) strBuilder.WriteString(".html") outputPath := filepath.Join(g.outputdir, strBuilder.String()) + if _, err := os.Stat(outputPath); err == nil { + return outputPath, nil + } + file, err := os.Create(outputPath) defer file.Close() if err != nil { - return errors.Wrap(err, "failed create html...") + return "", errors.Wrap(err, "failed create html...") } strBuilder.Reset() u, err := url.Parse(endpoint) if err != nil { - return err + return "", err } u.Path = path.Join(u.Path, "portal", "Public", "Syllabus", "SyllabusSearchStart.aspx") q := u.Query() - q.Set("lect_year", strconv.Itoa(g.year)) - q.Set("lect_cd", lectureID) + q.Set("lct_year", strconv.Itoa(g.year)) + q.Set("lct_cd", lectureID) q.Set("je_cd", "1") u.RawQuery = q.Encode() + fmt.Println(u.String()) res, err := http.Get(u.String()) defer res.Body.Close() if err != nil { - return errors.Wrap(err, "failed download html") + return "", errors.Wrap(err, "failed download html") } _, err = io.Copy(file, res.Body) if err != nil { - return errors.Wrap(err, "failed download html") + return "", errors.Wrap(err, "failed download html") } - return nil + return outputPath, nil } //LecIDwFilePath2LectureStruct is require LectureID (== Lecture.ID), filePath ( syllabus.html path) -func (g *GetSyllabus) LecIDwFilePath2LectureStruct(lectureID, filePath string) (*Lecture, error) { - file, err := os.Open(filePath) +func (g *GetSyllabus) LecIDwFilePath2LectureStruct(lwp *LectureWPath) (*Lecture, error) { + file, err := os.Open(lwp.Path) if err != nil { return nil, errors.Wrap(err, "failed open html file") @@ -152,7 +157,7 @@ scanner := bufio.NewScanner(file) var lec Lecture - lec.ID = lectureID + lec.ID = lwp.ID for scanner.Scan() { line := scanner.Text() @@ -162,15 +167,25 @@ if j := strings.Index(line, endSpan); j >= 0 { i += len(dayPeriodID) day := line[i:j] - lec.Day.DayOfWeek, err = kanjiday2int(day[0:dayOfWeeklen]) + + lec.Day.DayOfWeek = kanjiday2int(day[0:dayOfWeeklen]) if err != nil { return nil, errors.Wrap(err, "failed convert day") } lec.Day.Period, err = strconv.Atoi(day[dayOfWeeklen : dayOfWeeklen+1]) - - // dayの長さで〜があるかどうかが判定する + if err != nil { + return nil, errors.Wrap(err, "failed convert day") + } + if len(day) != (dayOfWeeklen + 1) { // dayOfWeeklen + 1 == 月3, 火2 + lec.Day.LastPeriod, err = strconv.Atoi(day[dayOfWeeklen+4:]) // 4 is \d + 〜 + if err != nil { + return nil, errors.Wrap(err, "failed convert day") + } + continue + } + lec.Day.LastPeriod = -1 + continue } - continue } // lecture name @@ -196,6 +211,18 @@ return &lec, nil } -func kanjiday2int(kanjiDay string) (int, error) { - return 0, nil +func kanjiday2int(kanjiDay string) int { + switch kanjiDay { + case "月": + return 0 + case "火": + return 1 + case "水": + return 2 + case "木": + return 3 + case "金": + return 4 + } + return -1 }