Merge pull request #116 from BITNP/laststarted

Add MirrorStatus.LastStarted property

internal/msg.go

@@ -8,16 +8,17 @@ import (
 // A MirrorStatus represents a msg when
 // a worker has done syncing
 type MirrorStatus struct {
-	Name       string     `json:"name"`
-	Worker     string     `json:"worker"`
-	IsMaster   bool       `json:"is_master"`
-	Status     SyncStatus `json:"status"`
-	LastUpdate time.Time  `json:"last_update"`
-	LastEnded  time.Time  `json:"last_ended"`
-	Scheduled  time.Time  `json:"next_schedule"`
-	Upstream   string     `json:"upstream"`
-	Size       string     `json:"size"`
-	ErrorMsg   string     `json:"error_msg"`
+	Name        string     `json:"name"`
+	Worker      string     `json:"worker"`
+	IsMaster    bool       `json:"is_master"`
+	Status      SyncStatus `json:"status"`
+	LastUpdate  time.Time  `json:"last_update"`
+	LastStarted time.Time  `json:"last_started"`
+	LastEnded   time.Time  `json:"last_ended"`
+	Scheduled   time.Time  `json:"next_schedule"`
+	Upstream    string     `json:"upstream"`
+	Size        string     `json:"size"`
+	ErrorMsg    string     `json:"error_msg"`
 }
 
 // A WorkerStatus is the information struct that describe

internal/status_web.go

@@ -38,31 +38,35 @@ func (t *stampTime) UnmarshalJSON(b []byte) error {
 
 // WebMirrorStatus is the mirror status to be shown in the web page
 type WebMirrorStatus struct {
-	Name         string     `json:"name"`
-	IsMaster     bool       `json:"is_master"`
-	Status       SyncStatus `json:"status"`
-	LastUpdate   textTime   `json:"last_update"`
-	LastUpdateTs stampTime  `json:"last_update_ts"`
-	LastEnded    textTime   `json:"last_ended"`
-	LastEndedTs  stampTime  `json:"last_ended_ts"`
-	Scheduled    textTime   `json:"next_schedule"`
-	ScheduledTs  stampTime  `json:"next_schedule_ts"`
-	Upstream     string     `json:"upstream"`
-	Size         string     `json:"size"` // approximate size
+	Name          string     `json:"name"`
+	IsMaster      bool       `json:"is_master"`
+	Status        SyncStatus `json:"status"`
+	LastUpdate    textTime   `json:"last_update"`
+	LastUpdateTs  stampTime  `json:"last_update_ts"`
+	LastStarted   textTime   `json:"last_started"`
+	LastStartedTs stampTime  `json:"last_started_ts"`
+	LastEnded     textTime   `json:"last_ended"`
+	LastEndedTs   stampTime  `json:"last_ended_ts"`
+	Scheduled     textTime   `json:"next_schedule"`
+	ScheduledTs   stampTime  `json:"next_schedule_ts"`
+	Upstream      string     `json:"upstream"`
+	Size          string     `json:"size"` // approximate size
 }
 
 func BuildWebMirrorStatus(m MirrorStatus) WebMirrorStatus {
 	return WebMirrorStatus{
-		Name:         m.Name,
-		IsMaster:     m.IsMaster,
-		Status:       m.Status,
-		LastUpdate:   textTime{m.LastUpdate},
-		LastUpdateTs: stampTime{m.LastUpdate},
-		LastEnded:    textTime{m.LastEnded},
-		LastEndedTs:  stampTime{m.LastEnded},
-		Scheduled:    textTime{m.Scheduled},
-		ScheduledTs:  stampTime{m.Scheduled},
-		Upstream:     m.Upstream,
-		Size:         m.Size,
+		Name:          m.Name,
+		IsMaster:      m.IsMaster,
+		Status:        m.Status,
+		LastUpdate:    textTime{m.LastUpdate},
+		LastUpdateTs:  stampTime{m.LastUpdate},
+		LastStarted:   textTime{m.LastStarted},
+		LastStartedTs: stampTime{m.LastStarted},
+		LastEnded:     textTime{m.LastEnded},
+		LastEndedTs:   stampTime{m.LastEnded},
+		Scheduled:     textTime{m.Scheduled},
+		ScheduledTs:   stampTime{m.Scheduled},
+		Upstream:      m.Upstream,
+		Size:          m.Size,
 	}
 }

internal/status_web_test.go

@@ -15,16 +15,18 @@ func TestStatus(t *testing.T) {
 		So(err, ShouldBeNil)
 		t := time.Date(2016, time.April, 16, 23, 8, 10, 0, loc)
 		m := WebMirrorStatus{
-			Name:         "tunalinux",
-			Status:       Success,
-			LastUpdate:   textTime{t},
-			LastUpdateTs: stampTime{t},
-			LastEnded:    textTime{t},
-			LastEndedTs:  stampTime{t},
-			Scheduled:    textTime{t},
-			ScheduledTs:  stampTime{t},
-			Size:         "5GB",
-			Upstream:     "rsync://mirrors.tuna.tsinghua.edu.cn/tunalinux/",
+			Name:          "tunalinux",
+			Status:        Success,
+			LastUpdate:    textTime{t},
+			LastUpdateTs:  stampTime{t},
+			LastStarted:   textTime{t},
+			LastStartedTs: stampTime{t},
+			LastEnded:     textTime{t},
+			LastEndedTs:   stampTime{t},
+			Scheduled:     textTime{t},
+			ScheduledTs:   stampTime{t},
+			Size:          "5GB",
+			Upstream:      "rsync://mirrors.tuna.tsinghua.edu.cn/tunalinux/",
 		}
 
 		b, err := json.Marshal(m)
@@ -40,6 +42,10 @@ func TestStatus(t *testing.T) {
 		So(m2.LastUpdateTs.Unix(), ShouldEqual, m.LastUpdate.Unix())
 		So(m2.LastUpdate.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
 		So(m2.LastUpdateTs.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
+		So(m2.LastStarted.Unix(), ShouldEqual, m.LastStarted.Unix())
+		So(m2.LastStartedTs.Unix(), ShouldEqual, m.LastStarted.Unix())
+		So(m2.LastStarted.UnixNano(), ShouldEqual, m.LastStarted.UnixNano())
+		So(m2.LastStartedTs.UnixNano(), ShouldEqual, m.LastStarted.UnixNano())
 		So(m2.LastEnded.Unix(), ShouldEqual, m.LastEnded.Unix())
 		So(m2.LastEndedTs.Unix(), ShouldEqual, m.LastEnded.Unix())
 		So(m2.LastEnded.UnixNano(), ShouldEqual, m.LastEnded.UnixNano())
@@ -53,15 +59,16 @@ func TestStatus(t *testing.T) {
 	})
 	Convey("BuildWebMirrorStatus should work", t, func() {
 		m := MirrorStatus{
-			Name:       "arch-sync3",
-			Worker:     "testWorker",
-			IsMaster:   true,
-			Status:     Failed,
-			LastUpdate: time.Now().Add(-time.Minute * 30),
-			LastEnded:  time.Now(),
-			Scheduled:  time.Now().Add(time.Minute * 5),
-			Upstream:   "mirrors.tuna.tsinghua.edu.cn",
-			Size:       "4GB",
+			Name:        "arch-sync3",
+			Worker:      "testWorker",
+			IsMaster:    true,
+			Status:      Failed,
+			LastUpdate:  time.Now().Add(-time.Minute * 30),
+			LastStarted: time.Now().Add(-time.Minute * 1),
+			LastEnded:   time.Now(),
+			Scheduled:   time.Now().Add(time.Minute * 5),
+			Upstream:    "mirrors.tuna.tsinghua.edu.cn",
+			Size:        "4GB",
 		}
 
 		var m2 WebMirrorStatus
@@ -73,6 +80,10 @@ func TestStatus(t *testing.T) {
 		So(m2.LastUpdateTs.Unix(), ShouldEqual, m.LastUpdate.Unix())
 		So(m2.LastUpdate.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
 		So(m2.LastUpdateTs.UnixNano(), ShouldEqual, m.LastUpdate.UnixNano())
+		So(m2.LastStarted.Unix(), ShouldEqual, m.LastStarted.Unix())
+		So(m2.LastStartedTs.Unix(), ShouldEqual, m.LastStarted.Unix())
+		So(m2.LastStarted.UnixNano(), ShouldEqual, m.LastStarted.UnixNano())
+		So(m2.LastStartedTs.UnixNano(), ShouldEqual, m.LastStarted.UnixNano())
 		So(m2.LastEnded.Unix(), ShouldEqual, m.LastEnded.Unix())
 		So(m2.LastEndedTs.Unix(), ShouldEqual, m.LastEnded.Unix())
 		So(m2.LastEnded.UnixNano(), ShouldEqual, m.LastEnded.UnixNano())

manager/db_test.go

@@ -78,34 +78,37 @@ func TestBoltAdapter(t *testing.T) {
 		Convey("update mirror status", func() {
 			status := []MirrorStatus{
 				MirrorStatus{
-					Name:       "arch-sync1",
-					Worker:     testWorkerIDs[0],
-					IsMaster:   true,
-					Status:     Success,
-					LastUpdate: time.Now(),
-					LastEnded:  time.Now(),
-					Upstream:   "mirrors.tuna.tsinghua.edu.cn",
-					Size:       "3GB",
+					Name:        "arch-sync1",
+					Worker:      testWorkerIDs[0],
+					IsMaster:    true,
+					Status:      Success,
+					LastUpdate:  time.Now(),
+					LastStarted: time.Now().Add(-time.Minute),
+					LastEnded:   time.Now(),
+					Upstream:    "mirrors.tuna.tsinghua.edu.cn",
+					Size:        "3GB",
 				},
 				MirrorStatus{
-					Name:       "arch-sync2",
-					Worker:     testWorkerIDs[1],
-					IsMaster:   true,
-					Status:     Disabled,
-					LastUpdate: time.Now().Add(-time.Hour),
-					LastEnded:  time.Now(),
-					Upstream:   "mirrors.tuna.tsinghua.edu.cn",
-					Size:       "4GB",
+					Name:        "arch-sync2",
+					Worker:      testWorkerIDs[1],
+					IsMaster:    true,
+					Status:      Disabled,
+					LastUpdate:  time.Now().Add(-time.Hour),
+					LastStarted: time.Now().Add(-time.Minute),
+					LastEnded:   time.Now(),
+					Upstream:    "mirrors.tuna.tsinghua.edu.cn",
+					Size:        "4GB",
 				},
 				MirrorStatus{
-					Name:       "arch-sync3",
-					Worker:     testWorkerIDs[1],
-					IsMaster:   true,
-					Status:     Success,
-					LastUpdate: time.Now().Add(-time.Second),
-					LastEnded:  time.Now(),
-					Upstream:   "mirrors.tuna.tsinghua.edu.cn",
-					Size:       "4GB",
+					Name:        "arch-sync3",
+					Worker:      testWorkerIDs[1],
+					IsMaster:    true,
+					Status:      Success,
+					LastUpdate:  time.Now().Add(-time.Minute),
+					LastStarted: time.Now().Add(-time.Second),
+					LastEnded:   time.Now(),
+					Upstream:    "mirrors.tuna.tsinghua.edu.cn",
+					Size:        "4GB",
 				},
 			}
 

manager/server.go

@@ -297,14 +297,21 @@ func (s *Manager) updateJobOfWorker(c *gin.Context) {
 
 	curStatus, _ := s.adapter.GetMirrorStatus(workerID, mirrorName)
 
+	curTime := time.Now()
+
+	if status.Status == PreSyncing && curStatus.Status != PreSyncing {
+		status.LastStarted = curTime
+	} else {
+		status.LastStarted = curStatus.LastStarted
+	}
 	// Only successful syncing needs last_update
 	if status.Status == Success {
-		status.LastUpdate = time.Now()
+		status.LastUpdate = curTime
 	} else {
 		status.LastUpdate = curStatus.LastUpdate
 	}
 	if status.Status == Success || status.Status == Failed {
-		status.LastEnded = time.Now()
+		status.LastEnded = curTime
 	} else {
 		status.LastEnded = curStatus.LastEnded
 	}

manager/server_test.go

@@ -151,10 +151,41 @@ func TestHTTPServer(t *testing.T) {
 					So(m.Size, ShouldEqual, status.Size)
 					So(m.IsMaster, ShouldEqual, status.IsMaster)
 					So(time.Now().Sub(m.LastUpdate), ShouldBeLessThan, 1*time.Second)
+					So(m.LastStarted.IsZero(), ShouldBeTrue) // hasn't been initialized yet
 					So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 1*time.Second)
 
 				})
 
+				// start syncing
+				status.Status = PreSyncing
+				time.Sleep(1 * time.Second)
+				resp, err = PostJSON(fmt.Sprintf("%s/workers/%s/jobs/%s", baseURL, status.Worker, status.Name), status, nil)
+				So(err, ShouldBeNil)
+				defer resp.Body.Close()
+				So(resp.StatusCode, ShouldEqual, http.StatusOK)
+
+				Convey("update mirror status to PreSync - starting sync", func(ctx C) {
+					var ms []MirrorStatus
+					resp, err := GetJSON(baseURL+"/workers/test_worker1/jobs", &ms, nil)
+
+					So(err, ShouldBeNil)
+					So(resp.StatusCode, ShouldEqual, http.StatusOK)
+					// err = json.NewDecoder(resp.Body).Decode(&mirrorStatusList)
+					m := ms[0]
+					So(m.Name, ShouldEqual, status.Name)
+					So(m.Worker, ShouldEqual, status.Worker)
+					So(m.Status, ShouldEqual, status.Status)
+					So(m.Upstream, ShouldEqual, status.Upstream)
+					So(m.Size, ShouldEqual, status.Size)
+					So(m.IsMaster, ShouldEqual, status.IsMaster)
+					So(time.Now().Sub(m.LastUpdate), ShouldBeLessThan, 3*time.Second)
+					So(time.Now().Sub(m.LastUpdate), ShouldBeGreaterThan, 1*time.Second)
+					So(time.Now().Sub(m.LastStarted), ShouldBeLessThan, 2*time.Second)
+					So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 3*time.Second)
+					So(time.Now().Sub(m.LastEnded), ShouldBeGreaterThan, 1*time.Second)
+
+				})
+
 				Convey("list all job status of all workers", func(ctx C) {
 					var ms []WebMirrorStatus
 					resp, err := GetJSON(baseURL+"/jobs", &ms, nil)
@@ -167,8 +198,9 @@ func TestHTTPServer(t *testing.T) {
 					So(m.Upstream, ShouldEqual, status.Upstream)
 					So(m.Size, ShouldEqual, status.Size)
 					So(m.IsMaster, ShouldEqual, status.IsMaster)
-					So(time.Now().Sub(m.LastUpdate.Time), ShouldBeLessThan, 1*time.Second)
-					So(time.Now().Sub(m.LastEnded.Time), ShouldBeLessThan, 1*time.Second)
+					So(time.Now().Sub(m.LastUpdate.Time), ShouldBeLessThan, 3*time.Second)
+					So(time.Now().Sub(m.LastStarted.Time), ShouldBeLessThan, 2*time.Second)
+					So(time.Now().Sub(m.LastEnded.Time), ShouldBeLessThan, 3*time.Second)
 
 				})
 
@@ -197,8 +229,9 @@ func TestHTTPServer(t *testing.T) {
 						So(m.Upstream, ShouldEqual, status.Upstream)
 						So(m.Size, ShouldEqual, "5GB")
 						So(m.IsMaster, ShouldEqual, status.IsMaster)
-						So(time.Now().Sub(m.LastUpdate), ShouldBeLessThan, 1*time.Second)
-						So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 1*time.Second)
+						So(time.Now().Sub(m.LastUpdate), ShouldBeLessThan, 3*time.Second)
+						So(time.Now().Sub(m.LastStarted), ShouldBeLessThan, 2*time.Second)
+						So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 3*time.Second)
 					})
 				})
 
@@ -251,6 +284,7 @@ func TestHTTPServer(t *testing.T) {
 					So(m.Size, ShouldEqual, status.Size)
 					So(m.IsMaster, ShouldEqual, status.IsMaster)
 					So(time.Now().Sub(m.LastUpdate), ShouldBeGreaterThan, 3*time.Second)
+					So(time.Now().Sub(m.LastStarted), ShouldBeGreaterThan, 3*time.Second)
 					So(time.Now().Sub(m.LastEnded), ShouldBeLessThan, 1*time.Second)
 				})
 			})
@@ -258,14 +292,15 @@ func TestHTTPServer(t *testing.T) {
 			Convey("update mirror status of an inexisted worker", func(ctx C) {
 				invalidWorker := "test_worker2"
 				status := MirrorStatus{
-					Name:       "arch-sync2",
-					Worker:     invalidWorker,
-					IsMaster:   true,
-					Status:     Success,
-					LastUpdate: time.Now(),
-					LastEnded:  time.Now(),
-					Upstream:   "mirrors.tuna.tsinghua.edu.cn",
-					Size:       "4GB",
+					Name:        "arch-sync2",
+					Worker:      invalidWorker,
+					IsMaster:    true,
+					Status:      Success,
+					LastUpdate:  time.Now(),
+					LastStarted: time.Now(),
+					LastEnded:   time.Now(),
+					Upstream:    "mirrors.tuna.tsinghua.edu.cn",
+					Size:        "4GB",
 				}
 				resp, err := PostJSON(fmt.Sprintf("%s/workers/%s/jobs/%s",
 					baseURL, status.Worker, status.Name), status, nil)