Merge pull request #80 from tuna/dev

Fix the "list" command of tunasynctl

cmd/tunasynctl/tunasynctl.go

@@ -140,8 +140,7 @@ func listWorkers(c *cli.Context) error {
 }
 
 func listJobs(c *cli.Context) error {
-	// FIXME: there should be an API on manager server side that return MirrorStatus list to tunasynctl
-	var jobs []tunasync.MirrorStatus
+	var jobs []tunasync.WebMirrorStatus
 	if c.Bool("all") {
 		_, err := tunasync.GetJSON(baseURL+listJobsPath, &jobs, client)
 		if err != nil {
@@ -158,10 +157,10 @@ func listJobs(c *cli.Context) error {
 				fmt.Sprintf("Usage Error: jobs command need at"+
 					" least one arguments or \"--all\" flag."), 1)
 		}
-		ans := make(chan []tunasync.MirrorStatus, len(args))
+		ans := make(chan []tunasync.WebMirrorStatus, len(args))
 		for _, workerID := range args {
 			go func(workerID string) {
-				var workerJobs []tunasync.MirrorStatus
+				var workerJobs []tunasync.WebMirrorStatus
 				_, err := tunasync.GetJSON(fmt.Sprintf("%s/workers/%s/jobs",
 					baseURL, workerID), &workerJobs, client)
 				if err != nil {

internal/msg.go

@@ -13,6 +13,7 @@ type MirrorStatus struct {
 	IsMaster   bool       `json:"is_master"`
 	Status     SyncStatus `json:"status"`
 	LastUpdate time.Time  `json:"last_update"`
+	LastEnded  time.Time  `json:"last_ended"`
 	Upstream   string     `json:"upstream"`
 	Size       string     `json:"size"`
 	ErrorMsg   string     `json:"error_msg"`

manager/status.go → internal/status_web.go

@@ -1,11 +1,9 @@
-package manager
+package internal
 
 import (
 	"encoding/json"
 	"strconv"
 	"time"
-
-	. "github.com/tuna/tunasync/internal"
 )
 
 type textTime struct {
@@ -38,24 +36,28 @@ func (t *stampTime) UnmarshalJSON(b []byte) error {
 	return err
 }
 
-// webMirrorStatus is the mirror status to be shown in the web page
-type webMirrorStatus struct {
+// WebMirrorStatus is the mirror status to be shown in the web page
+type WebMirrorStatus struct {
 	Name         string     `json:"name"`
 	IsMaster     bool       `json:"is_master"`
 	Status       SyncStatus `json:"status"`
 	LastUpdate   textTime   `json:"last_update"`
 	LastUpdateTs stampTime  `json:"last_update_ts"`
+	LastEnded    textTime   `json:"last_ended"`
+	LastEndedTs  stampTime  `json:"last_ended_ts"`
 	Upstream     string     `json:"upstream"`
 	Size         string     `json:"size"` // approximate size
 }
 
-func convertMirrorStatus(m MirrorStatus) webMirrorStatus {
-	return webMirrorStatus{
+func BuildWebMirrorStatus(m MirrorStatus) WebMirrorStatus {
+	return WebMirrorStatus{
 		Name:         m.Name,
 		IsMaster:     m.IsMaster,
 		Status:       m.Status,
 		LastUpdate:   textTime{m.LastUpdate},
 		LastUpdateTs: stampTime{m.LastUpdate},
+		LastEnded:    textTime{m.LastEnded},
+		LastEndedTs:  stampTime{m.LastEnded},
 		Upstream:     m.Upstream,
 		Size:         m.Size,
 	}

internal/status_web_test.go

@@ -0,0 +1,76 @@
+package internal
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestStatus(t *testing.T) {
+	Convey("status json ser-de should work", t, func() {
+		tz := "Asia/Tokyo"
+		loc, err := time.LoadLocation(tz)
+		So(err, ShouldBeNil)
+		t := time.Date(2016, time.April, 16, 23, 8, 10, 0, loc)
+		m := WebMirrorStatus{
+			Name:         "tunalinux",
+			Status:       Success,
+			LastUpdate:   textTime{t},
+			LastUpdateTs: stampTime{t},
+			LastEnded:    textTime{t},
+			LastEndedTs:  stampTime{t},
+			Size:         "5GB",
+			Upstream:     "rsync://mirrors.tuna.tsinghua.edu.cn/tunalinux/",
+		}
+
+		b, err := json.Marshal(m)
+		So(err, ShouldBeNil)
+		//fmt.Println(string(b))
+		var m2 WebMirrorStatus
+		err = json.Unmarshal(b, &m2)
+		So(err, ShouldBeNil)
+		// fmt.Printf("%#v", m2)
+		So(m2.Name, ShouldEqual, m.Name)
+		So(m2.Status, ShouldEqual, m.Status)
+		So(m2.LastUpdate.Unix(), ShouldEqual, m.LastUpdate.Unix())
+		So(m2.LastUpdateTs.Unix(), ShouldEqual, m.LastUpdate.Unix())
+		So(m2.LastUpdate.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
+		So(m2.LastUpdateTs.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
+		So(m2.LastEnded.Unix(), ShouldEqual, m.LastEnded.Unix())
+		So(m2.LastEndedTs.Unix(), ShouldEqual, m.LastEnded.Unix())
+		So(m2.LastEnded.UnixNano(), ShouldEqual, m.LastEnded.UnixNano())
+		So(m2.LastEndedTs.UnixNano(), ShouldEqual, m.LastEnded.UnixNano())
+		So(m2.Size, ShouldEqual, m.Size)
+		So(m2.Upstream, ShouldEqual, m.Upstream)
+	})
+	Convey("BuildWebMirrorStatus should work", t, func() {
+		m := MirrorStatus{
+			Name:       "arch-sync3",
+			Worker:     "testWorker",
+			IsMaster:   true,
+			Status:     Failed,
+			LastUpdate: time.Now().Add(-time.Minute * 30),
+			LastEnded:  time.Now(),
+			Upstream:   "mirrors.tuna.tsinghua.edu.cn",
+			Size:       "4GB",
+		}
+
+		var m2 WebMirrorStatus
+		m2 = BuildWebMirrorStatus(m)
+		// fmt.Printf("%#v", m2)
+		So(m2.Name, ShouldEqual, m.Name)
+		So(m2.Status, ShouldEqual, m.Status)
+		So(m2.LastUpdate.Unix(), ShouldEqual, m.LastUpdate.Unix())
+		So(m2.LastUpdateTs.Unix(), ShouldEqual, m.LastUpdate.Unix())
+		So(m2.LastUpdate.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
+		So(m2.LastUpdateTs.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
+		So(m2.LastEnded.Unix(), ShouldEqual, m.LastEnded.Unix())
+		So(m2.LastEndedTs.Unix(), ShouldEqual, m.LastEnded.Unix())
+		So(m2.LastEnded.UnixNano(), ShouldEqual, m.LastEnded.UnixNano())
+		So(m2.LastEndedTs.UnixNano(), ShouldEqual, m.LastEnded.UnixNano())
+		So(m2.Size, ShouldEqual, m.Size)
+		So(m2.Upstream, ShouldEqual, m.Upstream)
+	})
+}

manager/db_test.go

@@ -65,6 +65,7 @@ func TestBoltAdapter(t *testing.T) {
 					IsMaster:   true,
 					Status:     Success,
 					LastUpdate: time.Now(),
+					LastEnded:  time.Now(),
 					Upstream:   "mirrors.tuna.tsinghua.edu.cn",
 					Size:       "3GB",
 				},
@@ -73,7 +74,8 @@ func TestBoltAdapter(t *testing.T) {
 					Worker:     testWorkerIDs[1],
 					IsMaster:   true,
 					Status:     Disabled,
-					LastUpdate: time.Now(),
+					LastUpdate: time.Now().Add(-time.Hour),
+					LastEnded:  time.Now(),
 					Upstream:   "mirrors.tuna.tsinghua.edu.cn",
 					Size:       "4GB",
 				},
@@ -82,7 +84,8 @@ func TestBoltAdapter(t *testing.T) {
 					Worker:     testWorkerIDs[1],
 					IsMaster:   true,
 					Status:     Success,
-					LastUpdate: time.Now(),
+					LastUpdate: time.Now().Add(-time.Second),
+					LastEnded:  time.Now(),
 					Upstream:   "mirrors.tuna.tsinghua.edu.cn",
 					Size:       "4GB",
 				},

manager/server.go

@@ -135,11 +135,11 @@ func (s *Manager) listAllJobs(c *gin.Context) {
 		s.returnErrJSON(c, http.StatusInternalServerError, err)
 		return
 	}
-	webMirStatusList := []webMirrorStatus{}
+	webMirStatusList := []WebMirrorStatus{}
 	for _, m := range mirrorStatusList {
 		webMirStatusList = append(
 			webMirStatusList,
-			convertMirrorStatus(m),
+			BuildWebMirrorStatus(m),
 		)
 	}
 	c.JSON(http.StatusOK, webMirStatusList)
@@ -242,6 +242,11 @@ func (s *Manager) updateJobOfWorker(c *gin.Context) {
 	} else {
 		status.LastUpdate = curStatus.LastUpdate
 	}
+	if status.Status == Success || status.Status == Failed {
+		status.LastEnded = time.Now()
+	} else {
+		status.LastEnded = curStatus.LastEnded
+	}
 
 	// Only message with meaningful size updates the mirror size
 	if len(curStatus.Size) > 0 && curStatus.Size != "unknown" {

manager/server_test.go

@@ -121,11 +121,12 @@ func TestHTTPServer(t *testing.T) {
 					So(m.Size, ShouldEqual, status.Size)
 					So(m.IsMaster, ShouldEqual, status.IsMaster)
 					So(time.Now().Sub(m.LastUpdate), ShouldBeLessThan, 1*time.Second)
+					So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 1*time.Second)
 
 				})
 
 				Convey("list all job status of all workers", func(ctx C) {
-					var ms []webMirrorStatus
+					var ms []WebMirrorStatus
 					resp, err := GetJSON(baseURL+"/jobs", &ms, nil)
 					So(err, ShouldBeNil)
 					So(resp.StatusCode, ShouldEqual, http.StatusOK)
@@ -137,6 +138,7 @@ func TestHTTPServer(t *testing.T) {
 					So(m.Size, ShouldEqual, status.Size)
 					So(m.IsMaster, ShouldEqual, status.IsMaster)
 					So(time.Now().Sub(m.LastUpdate.Time), ShouldBeLessThan, 1*time.Second)
+					So(time.Now().Sub(m.LastEnded.Time), ShouldBeLessThan, 1*time.Second)
 
 				})
 
@@ -166,6 +168,7 @@ func TestHTTPServer(t *testing.T) {
 						So(m.Size, ShouldEqual, "5GB")
 						So(m.IsMaster, ShouldEqual, status.IsMaster)
 						So(time.Now().Sub(m.LastUpdate), ShouldBeLessThan, 1*time.Second)
+						So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 1*time.Second)
 					})
 				})
 
@@ -180,6 +183,32 @@ func TestHTTPServer(t *testing.T) {
 					So(err, ShouldBeNil)
 					So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError)
 				})
+
+				// what if status changed to failed
+				status.Status = Failed
+				time.Sleep(3 * time.Second)
+				resp, err = PostJSON(fmt.Sprintf("%s/workers/%s/jobs/%s", baseURL, status.Worker, status.Name), status, nil)
+				defer resp.Body.Close()
+				So(err, ShouldBeNil)
+				So(resp.StatusCode, ShouldEqual, http.StatusOK)
+
+				Convey("What if syncing job failed", func(ctx C) {
+					var ms []MirrorStatus
+					resp, err := GetJSON(baseURL+"/workers/test_worker1/jobs", &ms, nil)
+
+					So(err, ShouldBeNil)
+					So(resp.StatusCode, ShouldEqual, http.StatusOK)
+					// err = json.NewDecoder(resp.Body).Decode(&mirrorStatusList)
+					m := ms[0]
+					So(m.Name, ShouldEqual, status.Name)
+					So(m.Worker, ShouldEqual, status.Worker)
+					So(m.Status, ShouldEqual, status.Status)
+					So(m.Upstream, ShouldEqual, status.Upstream)
+					So(m.Size, ShouldEqual, status.Size)
+					So(m.IsMaster, ShouldEqual, status.IsMaster)
+					So(time.Now().Sub(m.LastUpdate), ShouldBeGreaterThan, 3*time.Second)
+					So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 1*time.Second)
+				})
 			})
 
 			Convey("update mirror status of an inexisted worker", func(ctx C) {
@@ -190,6 +219,7 @@ func TestHTTPServer(t *testing.T) {
 					IsMaster:   true,
 					Status:     Success,
 					LastUpdate: time.Now(),
+					LastEnded:  time.Now(),
 					Upstream:   "mirrors.tuna.tsinghua.edu.cn",
 					Size:       "4GB",
 				}

manager/status_test.go

@@ -1,44 +0,0 @@
-package manager
-
-import (
-	"encoding/json"
-	"testing"
-	"time"
-
-	tunasync "github.com/tuna/tunasync/internal"
-
-	. "github.com/smartystreets/goconvey/convey"
-)
-
-func TestStatus(t *testing.T) {
-	Convey("status json ser-de should work", t, func() {
-		tz := "Asia/Tokyo"
-		loc, err := time.LoadLocation(tz)
-		So(err, ShouldBeNil)
-		t := time.Date(2016, time.April, 16, 23, 8, 10, 0, loc)
-		m := webMirrorStatus{
-			Name:         "tunalinux",
-			Status:       tunasync.Success,
-			LastUpdate:   textTime{t},
-			LastUpdateTs: stampTime{t},
-			Size:         "5GB",
-			Upstream:     "rsync://mirrors.tuna.tsinghua.edu.cn/tunalinux/",
-		}
-
-		b, err := json.Marshal(m)
-		So(err, ShouldBeNil)
-		//fmt.Println(string(b))
-		var m2 webMirrorStatus
-		err = json.Unmarshal(b, &m2)
-		So(err, ShouldBeNil)
-		// fmt.Printf("%#v", m2)
-		So(m2.Name, ShouldEqual, m.Name)
-		So(m2.Status, ShouldEqual, m.Status)
-		So(m2.LastUpdate.Unix(), ShouldEqual, m.LastUpdate.Unix())
-		So(m2.LastUpdateTs.Unix(), ShouldEqual, m.LastUpdate.Unix())
-		So(m2.LastUpdate.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
-		So(m2.LastUpdateTs.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
-		So(m2.Size, ShouldEqual, m.Size)
-		So(m2.Upstream, ShouldEqual, m.Upstream)
-	})
-}