Merge pull request #114 from tuna/nest_mirror

Support nested mirror config

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/codeskyblue/go-sh v0.0.0-20190412065543-76bd3d59ff27
 	github.com/dennwc/btrfs v0.0.0-20190517175702-d917b30ff035
 	github.com/gin-gonic/gin v1.5.0
+	github.com/imdario/mergo v0.3.9
 	github.com/mattn/goveralls v0.0.5 // indirect
 	github.com/pkg/profile v1.4.0
 	github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46

go.sum

@@ -29,6 +29,9 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=
+github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
@@ -38,7 +41,9 @@ github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/goveralls v0.0.5 h1:spfq8AyZ0cCk57Za6/juJ5btQxeE1FaEGMdfcI+XO48=
 github.com/mattn/goveralls v0.0.5/go.mod h1:Xg2LHi51faXLyKXwsndxiW6uxEEQT9+3sjGzzwU4xy0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/pkg/profile v1.4.0 h1:uCmaf4vVbWAOZz36k1hrQD7ijGRzLwaME8Am/7a4jZI=
 github.com/pkg/profile v1.4.0/go.mod h1:NWz/XGvpEW1FyYQ7fCx4dqYBLlfTcE+A9FLAkNKqjFE=

worker/config.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 
 	"github.com/BurntSushi/toml"
+	"github.com/imdario/mergo"
 )
 
 type providerEnum uint8
@@ -41,7 +42,8 @@ type Config struct {
 	BtrfsSnapshot btrfsSnapshotConfig `toml:"btrfs_snapshot"`
 	Docker        dockerConfig        `toml:"docker"`
 	Include       includeConfig       `toml:"include"`
-	Mirrors       []mirrorConfig      `toml:"mirrors"`
+	MirrorsConf   []mirrorConfig      `toml:"mirrors"`
+	Mirrors       []mirrorConfig
 }
 
 type globalConfig struct {
@@ -111,15 +113,16 @@ type includedMirrorConfig struct {
 }
 
 type mirrorConfig struct {
-	Name      string            `toml:"name"`
-	Provider  providerEnum      `toml:"provider"`
-	Upstream  string            `toml:"upstream"`
-	Interval  int               `toml:"interval"`
-	Retry     int               `toml:"retry"`
-	MirrorDir string            `toml:"mirror_dir"`
-	LogDir    string            `toml:"log_dir"`
-	Env       map[string]string `toml:"env"`
-	Role      string            `toml:"role"`
+	Name         string            `toml:"name"`
+	Provider     providerEnum      `toml:"provider"`
+	Upstream     string            `toml:"upstream"`
+	Interval     int               `toml:"interval"`
+	Retry        int               `toml:"retry"`
+	MirrorDir    string            `toml:"mirror_dir"`
+	MirrorSubDir string            `toml:"mirror_subdir"`
+	LogDir       string            `toml:"log_dir"`
+	Env          map[string]string `toml:"env"`
+	Role         string            `toml:"role"`
 
 	// These two options over-write the global options
 	ExecOnSuccess []string `toml:"exec_on_success"`
@@ -148,6 +151,8 @@ type mirrorConfig struct {
 	DockerOptions []string `toml:"docker_options"`
 
 	SnapshotPath string `toml:"snapshot_path"`
+
+	ChildMirrors []mirrorConfig `toml:"mirrors"`
 }
 
 // LoadConfig loads configuration
@@ -174,9 +179,36 @@ func LoadConfig(cfgFile string) (*Config, error) {
 				logger.Errorf(err.Error())
 				return nil, err
 			}
-			cfg.Mirrors = append(cfg.Mirrors, incMirCfg.Mirrors...)
+			cfg.MirrorsConf = append(cfg.MirrorsConf, incMirCfg.Mirrors...)
+		}
+	}
+
+	for _, m := range cfg.MirrorsConf {
+		if err := recursiveMirrors(cfg, nil, m); err != nil {
+			return nil, err
 		}
 	}
 
 	return cfg, nil
 }
+
+func recursiveMirrors(cfg *Config, parent *mirrorConfig, mirror mirrorConfig) error {
+	var curMir mirrorConfig
+	if parent != nil {
+		curMir = *parent
+	}
+	curMir.ChildMirrors = nil
+	if err := mergo.Merge(&curMir, mirror, mergo.WithOverride); err != nil {
+		return err
+	}
+	if mirror.ChildMirrors == nil {
+		cfg.Mirrors = append(cfg.Mirrors, curMir)
+	} else {
+		for _, m := range mirror.ChildMirrors {
+			if err := recursiveMirrors(cfg, &curMir, m); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

worker/config_test.go

@@ -83,9 +83,9 @@ exec_on_failure = [
 			tmpDir,
 		)
 
-		cfgBlob = cfgBlob + incSection
+		curCfgBlob := cfgBlob + incSection
 
-		err = ioutil.WriteFile(tmpfile.Name(), []byte(cfgBlob), 0644)
+		err = ioutil.WriteFile(tmpfile.Name(), []byte(curCfgBlob), 0644)
 		So(err, ShouldEqual, nil)
 		defer tmpfile.Close()
 
@@ -157,6 +157,102 @@ use_ipv6 = true
 		So(len(cfg.Mirrors), ShouldEqual, 6)
 	})
 
+	Convey("Everything should work on nested config file", t, func() {
+		tmpfile, err := ioutil.TempFile("", "tunasync")
+		So(err, ShouldEqual, nil)
+		defer os.Remove(tmpfile.Name())
+
+		tmpDir, err := ioutil.TempDir("", "tunasync")
+		So(err, ShouldBeNil)
+		defer os.RemoveAll(tmpDir)
+
+		incSection := fmt.Sprintf(
+			"\n[include]\n"+
+				"include_mirrors = \"%s/*.conf\"",
+			tmpDir,
+		)
+
+		curCfgBlob := cfgBlob + incSection
+
+		err = ioutil.WriteFile(tmpfile.Name(), []byte(curCfgBlob), 0644)
+		So(err, ShouldEqual, nil)
+		defer tmpfile.Close()
+
+		incBlob1 := `
+[[mirrors]]
+name = "ipv6s"
+use_ipv6 = true
+	[[mirrors.mirrors]]
+	name = "debians"
+	mirror_subdir = "debian"
+	provider = "two-stage-rsync"
+	stage1_profile = "debian"
+
+		[[mirrors.mirrors.mirrors]]
+		name = "debian-security"
+		upstream = "rsync://test.host/debian-security/"
+		[[mirrors.mirrors.mirrors]]
+		name = "ubuntu"
+		stage1_profile = "ubuntu"
+		upstream = "rsync://test.host2/ubuntu/"
+	[[mirrors.mirrors]]
+	name = "debian-cd"
+	provider = "rsync"
+	upstream = "rsync://test.host3/debian-cd/"
+		`
+		err = ioutil.WriteFile(filepath.Join(tmpDir, "nest.conf"), []byte(incBlob1), 0644)
+		So(err, ShouldEqual, nil)
+
+		cfg, err := LoadConfig(tmpfile.Name())
+		So(err, ShouldBeNil)
+		So(cfg.Global.Name, ShouldEqual, "test_worker")
+		So(cfg.Global.Interval, ShouldEqual, 240)
+		So(cfg.Global.Retry, ShouldEqual, 3)
+		So(cfg.Global.MirrorDir, ShouldEqual, "/data/mirrors")
+
+		So(cfg.Manager.APIBase, ShouldEqual, "https://127.0.0.1:5000")
+		So(cfg.Server.Hostname, ShouldEqual, "worker1.example.com")
+
+		m := cfg.Mirrors[0]
+		So(m.Name, ShouldEqual, "AOSP")
+		So(m.MirrorDir, ShouldEqual, "/data/git/AOSP")
+		So(m.Provider, ShouldEqual, provCommand)
+		So(m.Interval, ShouldEqual, 720)
+		So(m.Retry, ShouldEqual, 2)
+		So(m.Env["REPO"], ShouldEqual, "/usr/local/bin/aosp-repo")
+
+		m = cfg.Mirrors[1]
+		So(m.Name, ShouldEqual, "debian")
+		So(m.MirrorDir, ShouldEqual, "")
+		So(m.Provider, ShouldEqual, provTwoStageRsync)
+
+		m = cfg.Mirrors[2]
+		So(m.Name, ShouldEqual, "fedora")
+		So(m.MirrorDir, ShouldEqual, "")
+		So(m.Provider, ShouldEqual, provRsync)
+		So(m.ExcludeFile, ShouldEqual, "/etc/tunasync.d/fedora-exclude.txt")
+
+		m = cfg.Mirrors[3]
+		So(m.Name, ShouldEqual, "debian-security")
+		So(m.MirrorDir, ShouldEqual, "")
+		So(m.Provider, ShouldEqual, provTwoStageRsync)
+		So(m.UseIPv6, ShouldEqual, true)
+		So(m.Stage1Profile, ShouldEqual, "debian")
+
+		m = cfg.Mirrors[4]
+		So(m.Name, ShouldEqual, "ubuntu")
+		So(m.MirrorDir, ShouldEqual, "")
+		So(m.Provider, ShouldEqual, provTwoStageRsync)
+		So(m.UseIPv6, ShouldEqual, true)
+		So(m.Stage1Profile, ShouldEqual, "ubuntu")
+
+		m = cfg.Mirrors[5]
+		So(m.Name, ShouldEqual, "debian-cd")
+		So(m.UseIPv6, ShouldEqual, true)
+		So(m.Provider, ShouldEqual, provRsync)
+
+		So(len(cfg.Mirrors), ShouldEqual, 6)
+	})
 	Convey("Providers can be inited from a valid config file", t, func() {
 		tmpfile, err := ioutil.TempFile("", "tunasync")
 		So(err, ShouldEqual, nil)
@@ -207,4 +303,90 @@ use_ipv6 = true
 		So(rp.excludeFile, ShouldEqual, "/etc/tunasync.d/fedora-exclude.txt")
 
 	})
+
+	Convey("MirrorSubdir should work", t, func() {
+		tmpfile, err := ioutil.TempFile("", "tunasync")
+		So(err, ShouldEqual, nil)
+		defer os.Remove(tmpfile.Name())
+
+		cfgBlob1 := `
+[global]
+name = "test_worker"
+log_dir = "/var/log/tunasync/{{.Name}}"
+mirror_dir = "/data/mirrors"
+concurrent = 10
+interval = 240
+retry = 3
+
+[manager]
+api_base = "https://127.0.0.1:5000"
+token = "some_token"
+
+[server]
+hostname = "worker1.example.com"
+listen_addr = "127.0.0.1"
+listen_port = 6000
+ssl_cert = "/etc/tunasync.d/worker1.cert"
+ssl_key = "/etc/tunasync.d/worker1.key"
+
+[[mirrors]]
+name = "ipv6s"
+use_ipv6 = true
+	[[mirrors.mirrors]]
+	name = "debians"
+	mirror_subdir = "debian"
+	provider = "two-stage-rsync"
+	stage1_profile = "debian"
+
+		[[mirrors.mirrors.mirrors]]
+		name = "debian-security"
+		upstream = "rsync://test.host/debian-security/"
+		[[mirrors.mirrors.mirrors]]
+		name = "ubuntu"
+		stage1_profile = "ubuntu"
+		upstream = "rsync://test.host2/ubuntu/"
+	[[mirrors.mirrors]]
+	name = "debian-cd"
+	provider = "rsync"
+	upstream = "rsync://test.host3/debian-cd/"
+		`
+		err = ioutil.WriteFile(tmpfile.Name(), []byte(cfgBlob1), 0644)
+		So(err, ShouldEqual, nil)
+		defer tmpfile.Close()
+
+		cfg, err := LoadConfig(tmpfile.Name())
+		So(err, ShouldBeNil)
+
+		providers := map[string]mirrorProvider{}
+		for _, m := range cfg.Mirrors {
+			p := newMirrorProvider(m, cfg)
+			providers[p.Name()] = p
+		}
+
+		p := providers["debian-security"]
+		So(p.Name(), ShouldEqual, "debian-security")
+		So(p.LogDir(), ShouldEqual, "/var/log/tunasync/debian-security")
+		So(p.LogFile(), ShouldEqual, "/var/log/tunasync/debian-security/latest.log")
+		r2p, ok := p.(*twoStageRsyncProvider)
+		So(ok, ShouldBeTrue)
+		So(r2p.stage1Profile, ShouldEqual, "debian")
+		So(r2p.WorkingDir(), ShouldEqual, "/data/mirrors/debian/debian-security")
+
+		p = providers["ubuntu"]
+		So(p.Name(), ShouldEqual, "ubuntu")
+		So(p.LogDir(), ShouldEqual, "/var/log/tunasync/ubuntu")
+		So(p.LogFile(), ShouldEqual, "/var/log/tunasync/ubuntu/latest.log")
+		r2p, ok = p.(*twoStageRsyncProvider)
+		So(ok, ShouldBeTrue)
+		So(r2p.stage1Profile, ShouldEqual, "ubuntu")
+		So(r2p.WorkingDir(), ShouldEqual, "/data/mirrors/debian/ubuntu")
+
+		p = providers["debian-cd"]
+		So(p.Name(), ShouldEqual, "debian-cd")
+		So(p.LogDir(), ShouldEqual, "/var/log/tunasync/debian-cd")
+		So(p.LogFile(), ShouldEqual, "/var/log/tunasync/debian-cd/latest.log")
+		rp, ok := p.(*rsyncProvider)
+		So(ok, ShouldBeTrue)
+		So(rp.WorkingDir(), ShouldEqual, "/data/mirrors/debian-cd")
+	})
 }

worker/provider.go

@@ -82,7 +82,7 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
 	}
 	if mirrorDir == "" {
 		mirrorDir = filepath.Join(
-			cfg.Global.MirrorDir, mirror.Name,
+			cfg.Global.MirrorDir, mirror.MirrorSubDir, mirror.Name,
 		)
 	}
 	if mirror.Interval == 0 {