feature(worker): implemented multi-file configrations. closing #23

tests/mirrors/aosp.conf

@@ -0,0 +1,12 @@
+[[mirrors]]
+name = "AOSP"
+provider = "command"
+command = "/tmp/tunasync/bin/myrsync2.sh"
+upstream = "https://aosp.google.com/"
+interval = 2
+mirror_dir = "/tmp/tunasync/git/AOSP"
+role = "slave"
+	[mirrors.env]
+	REPO = "/usr/local/bin/aosp-repo"
+
+# vim: ft=toml

tests/mirrors/deb_fedora.conf

@@ -0,0 +1,18 @@
+[[mirrors]]
+name = "debian"
+command = "/tmp/tunasync/bin/myrsync.sh"
+provider = "two-stage-rsync"
+stage1_profile = "debian"
+upstream = "rsync://ftp.debian.org/debian/"
+use_ipv6 = true
+
+
+[[mirrors]]
+name = "fedora"
+command = "/tmp/tunasync/bin/myrsync.sh"
+provider = "rsync"
+upstream = "rsync://ftp.fedoraproject.org/fedora/"
+use_ipv6 = true
+exclude_file = "/etc/tunasync.d/fedora-exclude.txt"
+
+# vim: ft=toml

tests/worker.conf

@@ -22,33 +22,7 @@ listen_port = 6000
 ssl_cert = "worker.crt"
 ssl_key = "worker.key"
 
-[[mirrors]]
-name = "AOSP"
-provider = "command"
-command = "/tmp/tunasync/bin/myrsync2.sh"
-upstream = "https://aosp.google.com/"
-interval = 2
-mirror_dir = "/tmp/tunasync/git/AOSP"
-role = "slave"
-	[mirrors.env]
-	REPO = "/usr/local/bin/aosp-repo"
-
-[[mirrors]]
-name = "debian"
-command = "/tmp/tunasync/bin/myrsync.sh"
-provider = "two-stage-rsync"
-stage1_profile = "debian"
-upstream = "rsync://ftp.debian.org/debian/"
-use_ipv6 = true
-
-
-[[mirrors]]
-name = "fedora"
-command = "/tmp/tunasync/bin/myrsync.sh"
-provider = "rsync"
-upstream = "rsync://ftp.fedoraproject.org/fedora/"
-use_ipv6 = true
-exclude_file = "/etc/tunasync.d/fedora-exclude.txt"
-
+[include]
+include_mirrors = "mirrors/*.conf"
 
 # vim: ft=toml

worker/config.go

@@ -3,6 +3,7 @@ package worker
 import (
 	"errors"
 	"os"
+	"path/filepath"
 
 	"github.com/BurntSushi/toml"
 )
@@ -30,12 +31,13 @@ func (p *providerEnum) UnmarshalText(text []byte) error {
 	return nil
 }
 
-// Worker config options
+// Config represents worker config options
 type Config struct {
 	Global  globalConfig   `toml:"global"`
 	Manager managerConfig  `toml:"manager"`
 	Server  serverConfig   `toml:"server"`
 	Cgroup  cgroupConfig   `toml:"cgroup"`
+	Include includeConfig  `toml:"include"`
 	Mirrors []mirrorConfig `toml:"mirrors"`
 }
 
@@ -67,6 +69,14 @@ type cgroupConfig struct {
 	Group    string `toml:"group"`
 }
 
+type includeConfig struct {
+	IncludeMirrors string `toml:"include_mirrors"`
+}
+
+type includedMirrorConfig struct {
+	Mirrors []mirrorConfig `toml:"mirrors"`
+}
+
 type mirrorConfig struct {
 	Name      string            `toml:"name"`
 	Provider  providerEnum      `toml:"provider"`
@@ -98,5 +108,22 @@ func LoadConfig(cfgFile string) (*Config, error) {
 		logger.Errorf(err.Error())
 		return nil, err
 	}
+
+	if cfg.Include.IncludeMirrors != "" {
+		includedFiles, err := filepath.Glob(cfg.Include.IncludeMirrors)
+		if err != nil {
+			logger.Errorf(err.Error())
+			return nil, err
+		}
+		for _, f := range includedFiles {
+			var incMirCfg includedMirrorConfig
+			if _, err := toml.DecodeFile(f, &incMirCfg); err != nil {
+				logger.Errorf(err.Error())
+				return nil, err
+			}
+			cfg.Mirrors = append(cfg.Mirrors, incMirCfg.Mirrors...)
+		}
+	}
+
 	return cfg, nil
 }

worker/config_test.go

@@ -1,8 +1,10 @@
 package worker
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"testing"
 
 	. "github.com/smartystreets/goconvey/convey"
@@ -45,7 +47,6 @@ stage1_profile = "debian"
 upstream = "rsync://ftp.debian.org/debian/"
 use_ipv6 = true
 
-
 [[mirrors]]
 name = "fedora"
 provider = "rsync"
@@ -66,10 +67,47 @@ exec_on_failure = "bash -c 'echo ${TUNASYNC_JOB_EXIT_STATUS} > ${TUNASYNC_WORKIN
 		So(err, ShouldEqual, nil)
 		defer os.Remove(tmpfile.Name())
 
+		tmpDir, err := ioutil.TempDir("", "tunasync")
+		So(err, ShouldBeNil)
+		defer os.RemoveAll(tmpDir)
+
+		incSection := fmt.Sprintf(
+			"\n[include]\n"+
+				"include_mirrors = \"%s/*.conf\"",
+			tmpDir,
+		)
+
+		cfgBlob = cfgBlob + incSection
+
 		err = ioutil.WriteFile(tmpfile.Name(), []byte(cfgBlob), 0644)
 		So(err, ShouldEqual, nil)
 		defer tmpfile.Close()
 
+		incBlob1 := `
+[[mirrors]]
+name = "debian-cd"
+provider = "two-stage-rsync"
+stage1_profile = "debian"
+use_ipv6 = true
+
+[[mirrors]]
+name = "debian-security"
+provider = "two-stage-rsync"
+stage1_profile = "debian"
+use_ipv6 = true
+		`
+		incBlob2 := `
+[[mirrors]]
+name = "ubuntu"
+provider = "two-stage-rsync"
+stage1_profile = "debian"
+use_ipv6 = true
+		`
+		err = ioutil.WriteFile(filepath.Join(tmpDir, "debian.conf"), []byte(incBlob1), 0644)
+		So(err, ShouldEqual, nil)
+		err = ioutil.WriteFile(filepath.Join(tmpDir, "ubuntu.conf"), []byte(incBlob2), 0644)
+		So(err, ShouldEqual, nil)
+
 		cfg, err := LoadConfig(tmpfile.Name())
 		So(err, ShouldBeNil)
 		So(cfg.Global.Name, ShouldEqual, "test_worker")
@@ -97,7 +135,18 @@ exec_on_failure = "bash -c 'echo ${TUNASYNC_JOB_EXIT_STATUS} > ${TUNASYNC_WORKIN
 		So(m.Provider, ShouldEqual, provRsync)
 		So(m.ExcludeFile, ShouldEqual, "/etc/tunasync.d/fedora-exclude.txt")
 
-		So(len(cfg.Mirrors), ShouldEqual, 3)
+		m = cfg.Mirrors[3]
+		So(m.Name, ShouldEqual, "debian-cd")
+		So(m.MirrorDir, ShouldEqual, "")
+		So(m.Provider, ShouldEqual, provTwoStageRsync)
+
+		m = cfg.Mirrors[4]
+		So(m.Name, ShouldEqual, "debian-security")
+
+		m = cfg.Mirrors[5]
+		So(m.Name, ShouldEqual, "ubuntu")
+
+		So(len(cfg.Mirrors), ShouldEqual, 6)
 	})
 
 	Convey("Providers can be inited from a valid config file", t, func() {