# HG changeset patch # User anatofuz # Date 1585629606 -32400 # Node ID a0d23f38344d0c9536b9efa188ac528e48ecfb0b # Parent 8bc574052fcb8d768f18d581851eff827adfb43b ... diff -r 8bc574052fcb -r a0d23f38344d cmd_donwload.go --- a/cmd_donwload.go Tue Mar 31 12:47:04 2020 +0900 +++ b/cmd_donwload.go Tue Mar 31 13:40:06 2020 +0900 @@ -4,6 +4,8 @@ "context" "fmt" "io" + + "ie.u-ryukyu.ac.jp/hg/y19/index.cgi/home/hg/y19/k198584/Tools/lectable/syllabus" ) type cmdDownload struct{} @@ -18,5 +20,13 @@ func (cd *cmdDownload) run(ctx context.Context, argv []string, outStream, errStream io.Writer) error { fmt.Println("download now!!") - return nil + dh := syllabus.CreateGetSyllabus() + _, err := dh.CheckAndMkdirBuilddir() + if err != nil { + return err + } + arr3 := []string{"601495001", "600625001"} + + err = dh.LecIDStoDonwlodSyllabus(ctx, arr3) + return err } diff -r 8bc574052fcb -r a0d23f38344d go.sum --- a/go.sum Tue Mar 31 12:47:04 2020 +0900 +++ b/go.sum Tue Mar 31 13:40:06 2020 +0900 @@ -1,3 +1,4 @@ +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff -r 8bc574052fcb -r a0d23f38344d syllabus/getSyllabus.go --- a/syllabus/getSyllabus.go Tue Mar 31 12:47:04 2020 +0900 +++ b/syllabus/getSyllabus.go Tue Mar 31 13:40:06 2020 +0900 @@ -2,6 +2,7 @@ import ( "bufio" + "context" "io" "net/http" "net/url" @@ -10,6 +11,7 @@ "path/filepath" "strconv" "strings" + "sync" "time" "github.com/pkg/errors" @@ -41,7 +43,7 @@ type Lecture struct { ID string Name string - Day *lectureDay + Day *LectureDay Teacher string } @@ -71,19 +73,44 @@ //"https://tiglon.jim.u-ryukyu.ac.jp/portal/Public/Syllabus/SyllabusSearchStart.aspx?lct_year=2019&lct_cd=610004071&je_cd=1" var endpoint = "https://tiglon.jim.u-ryukyu.ac.jp" +//CheckAndMkdirBuilddir is builld 2019/early dir +func (g *GetSyllabus) CheckAndMkdirBuilddir() (bool, error) { + if f, err := os.Stat(g.outputdir); os.IsNotExist(err) || !f.IsDir() { + err := os.MkdirAll(g.outputdir, 0755) + if err != nil { + return false, errors.Wrap(err, "failed mkdir") + } + return true, nil + } + return true, nil +} + +func (g *GetSyllabus) LecIDStoDonwlodSyllabus(ctx context.Context, lectureIDs []string) error { + var wg sync.WaitGroup + for _, id := range lectureIDs { + wg.Add(1) + go func(id string) { + defer wg.Done() + g.LecIDtoDownloadSyllabus(id) + }(id) + } + wg.Wait() + return nil +} + //LecIDtoDownloadSyllabus is download from lecture ID func (g *GetSyllabus) LecIDtoDownloadSyllabus(lectureID string) error { var strBuilder strings.Builder strBuilder.WriteString(lectureID) strBuilder.WriteString(".html") - putputPath := filepath.Join(g.outputdir, strBuilder.String()) + outputPath := filepath.Join(g.outputdir, strBuilder.String()) - file, err := os.Create(strBuilder.String()) + file, err := os.Create(outputPath) defer file.Close() if err != nil { - return errors.Wrap("failed create html...") + return errors.Wrap(err, "failed create html...") } strBuilder.Reset() @@ -93,9 +120,9 @@ return err } - u.path = path.Join(u.path, "portal", "Public", "Syllabus", "SyllabusSearchStart.aspx") + u.Path = path.Join(u.Path, "portal", "Public", "Syllabus", "SyllabusSearchStart.aspx") q := u.Query() - q.Set("lect_year", g.year) + q.Set("lect_year", strconv.Itoa(g.year)) q.Set("lect_cd", lectureID) q.Set("je_cd", "1") u.RawQuery = q.Encode() @@ -104,19 +131,19 @@ defer res.Body.Close() if err != nil { - return errors.Wrap("failed download html") + return errors.Wrap(err, "failed download html") } - _, err := io.Copy(file, res.Body) + _, err = io.Copy(file, res.Body) if err != nil { - return errors.Wrap("failed download html") + return errors.Wrap(err, "failed download html") } return nil } //LecIDwFilePath2LectureStruct is require LectureID (== Lecture.ID), filePath ( syllabus.html path) -func (g *GetSyllabus) LecIDwFilePath2LectureStruct(lectureID, filePath string) (*lecture, error) { +func (g *GetSyllabus) LecIDwFilePath2LectureStruct(lectureID, filePath string) (*Lecture, error) { file, err := os.Open(filePath) if err != nil { @@ -124,8 +151,8 @@ } scanner := bufio.NewScanner(file) - var lec lecture - lec.id = lectureID + var lec Lecture + lec.ID = lectureID for scanner.Scan() { line := scanner.Text() @@ -135,11 +162,11 @@ if j := strings.Index(line, endSpan); j >= 0 { i += len(dayPeriodID) day := line[i:j] - lec.day.dayOfWeek, err = kanjiday2int(day[0:dayOfWeeklen]) + lec.Day.DayOfWeek, err = kanjiday2int(day[0:dayOfWeeklen]) if err != nil { return nil, errors.Wrap(err, "failed convert day") } - lec.day.period, err = strconv.Atoi(day[dayOfWeeklen : dayOfWeeklen+1]) + lec.Day.Period, err = strconv.Atoi(day[dayOfWeeklen : dayOfWeeklen+1]) // dayの長さで〜があるかどうかが判定する } @@ -150,7 +177,7 @@ if i := strings.Index(line, lectureNameID); i >= 0 { if j := strings.Index(line, endSpan); j >= 0 { i += len(lectureNameID) - lec.name = line[i:j] + lec.Name = line[i:j] } continue } @@ -159,7 +186,7 @@ if i := strings.Index(line, teacherNameID); i >= 0 { if j := strings.Index(line, endSpan); j >= 0 { i += len(teacherNameID) - lec.teacher = line[i:j] + lec.Teacher = line[i:j] } break }