Merge pull request #208 from tuna/rsync-opts

More configuration options for rsync

worker/config.go

@@ -58,6 +58,9 @@ type globalConfig struct {
 	Retry      int    `toml:"retry"`
 	Timeout    int    `toml:"timeout"`
 
+	// appended to the options generated by rsync_provider, but before mirror-specific options
+	RsyncOptions []string `toml:"rsync_options"`
+
 	ExecOnSuccess []string `toml:"exec_on_success"`
 	ExecOnFailure []string `toml:"exec_on_failure"`
 }
@@ -162,23 +165,24 @@ type mirrorConfig struct {
 	ExecOnSuccess []string `toml:"exec_on_success"`
 	ExecOnFailure []string `toml:"exec_on_failure"`
 
-	// These two options  the global options
+	// These two options are appended to the global options
 	ExecOnSuccessExtra []string `toml:"exec_on_success_extra"`
 	ExecOnFailureExtra []string `toml:"exec_on_failure_extra"`
 
-	Command       string   `toml:"command"`
-	FailOnMatch   string   `toml:"fail_on_match"`
-	SizePattern   string   `toml:"size_pattern"`
-	UseIPv6       bool     `toml:"use_ipv6"`
-	UseIPv4       bool     `toml:"use_ipv4"`
-	ExcludeFile   string   `toml:"exclude_file"`
-	Username      string   `toml:"username"`
-	Password      string   `toml:"password"`
-	RsyncNoTimeo  bool     `toml:"rsync_no_timeout"`
-	RsyncTimeout  int      `toml:"rsync_timeout"`
-	RsyncOptions  []string `toml:"rsync_options"`
-	RsyncOverride []string `toml:"rsync_override"`
-	Stage1Profile string   `toml:"stage1_profile"`
+	Command           string   `toml:"command"`
+	FailOnMatch       string   `toml:"fail_on_match"`
+	SizePattern       string   `toml:"size_pattern"`
+	UseIPv6           bool     `toml:"use_ipv6"`
+	UseIPv4           bool     `toml:"use_ipv4"`
+	ExcludeFile       string   `toml:"exclude_file"`
+	Username          string   `toml:"username"`
+	Password          string   `toml:"password"`
+	RsyncNoTimeo      bool     `toml:"rsync_no_timeout"`
+	RsyncTimeout      int      `toml:"rsync_timeout"`
+	RsyncOptions      []string `toml:"rsync_options"`
+	RsyncOverride     []string `toml:"rsync_override"`
+	RsyncOverrideOnly bool     `toml:"rsync_override_only"` // only use provided overridden options if true
+	Stage1Profile     string   `toml:"stage1_profile"`
 
 	MemoryLimit MemBytes `toml:"memory_limit"`
 

worker/config_test.go

@@ -403,4 +403,122 @@ use_ipv6 = true
 		So(rp.WorkingDir(), ShouldEqual, "/data/mirrors/debian-cd")
 		So(p.Timeout(), ShouldEqual, 86400*time.Second)
 	})
+
+	Convey("rsync_override_only should work", t, func() {
+		tmpfile, err := os.CreateTemp("", "tunasync")
+		So(err, ShouldEqual, nil)
+		defer os.Remove(tmpfile.Name())
+
+		cfgBlob1 := `
+[global]
+name = "test_worker"
+log_dir = "/var/log/tunasync/{{.Name}}"
+mirror_dir = "/data/mirrors"
+concurrent = 10
+interval = 240
+retry = 3
+timeout = 86400
+
+[manager]
+api_base = "https://127.0.0.1:5000"
+token = "some_token"
+
+[server]
+hostname = "worker1.example.com"
+listen_addr = "127.0.0.1"
+listen_port = 6000
+ssl_cert = "/etc/tunasync.d/worker1.cert"
+ssl_key = "/etc/tunasync.d/worker1.key"
+
+[[mirrors]]
+name = "foo"
+provider = "rsync"
+upstream = "rsync://foo.bar/"
+interval = 720
+retry = 2
+timeout = 3600
+mirror_dir = "/data/foo"
+rsync_override = ["--bar", "baz"]
+rsync_override_only = true
+`
+
+		err = os.WriteFile(tmpfile.Name(), []byte(cfgBlob1), 0644)
+		So(err, ShouldEqual, nil)
+		defer tmpfile.Close()
+
+		cfg, err := LoadConfig(tmpfile.Name())
+		So(err, ShouldBeNil)
+
+		providers := map[string]mirrorProvider{}
+		for _, m := range cfg.Mirrors {
+			p := newMirrorProvider(m, cfg)
+			providers[p.Name()] = p
+		}
+
+		p, ok := providers["foo"].(*rsyncProvider)
+		So(ok, ShouldBeTrue)
+		So(p.options, ShouldResemble, []string{"--bar", "baz"})
+	})
+
+	Convey("rsync global options should work", t, func() {
+		tmpfile, err := os.CreateTemp("", "tunasync")
+		So(err, ShouldEqual, nil)
+		defer os.Remove(tmpfile.Name())
+
+		cfgBlob1 := `
+[global]
+name = "test_worker"
+log_dir = "/var/log/tunasync/{{.Name}}"
+mirror_dir = "/data/mirrors"
+concurrent = 10
+interval = 240
+retry = 3
+timeout = 86400
+rsync_options = ["--global"]
+
+[manager]
+api_base = "https://127.0.0.1:5000"
+token = "some_token"
+
+[server]
+hostname = "worker1.example.com"
+listen_addr = "127.0.0.1"
+listen_port = 6000
+ssl_cert = "/etc/tunasync.d/worker1.cert"
+ssl_key = "/etc/tunasync.d/worker1.key"
+
+[[mirrors]]
+name = "foo"
+provider = "rsync"
+upstream = "rsync://foo.bar/"
+interval = 720
+retry = 2
+timeout = 3600
+mirror_dir = "/data/foo"
+rsync_override = ["--override"]
+rsync_options = ["--local"]
+`
+
+		err = os.WriteFile(tmpfile.Name(), []byte(cfgBlob1), 0644)
+		So(err, ShouldEqual, nil)
+		defer tmpfile.Close()
+
+		cfg, err := LoadConfig(tmpfile.Name())
+		So(err, ShouldBeNil)
+
+		providers := map[string]mirrorProvider{}
+		for _, m := range cfg.Mirrors {
+			p := newMirrorProvider(m, cfg)
+			providers[p.Name()] = p
+		}
+
+		p, ok := providers["foo"].(*rsyncProvider)
+		So(ok, ShouldBeTrue)
+		So(p.options, ShouldResemble, []string{
+			"--override",    // from mirror.rsync_override
+			"--timeout=120", // generated by newRsyncProvider
+			"--global",      // from global.rsync_options
+			"--local",       // from mirror.rsync_options
+		})
+	})
 }

worker/provider.go

@@ -142,7 +142,9 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
 			extraOptions:      mirror.RsyncOptions,
 			rsyncNeverTimeout: mirror.RsyncNoTimeo,
 			rsyncTimeoutValue: mirror.RsyncTimeout,
+			globalOptions:     cfg.Global.RsyncOptions,
 			overriddenOptions: mirror.RsyncOverride,
+			useOverrideOnly:   mirror.RsyncOverrideOnly,
 			rsyncEnv:          mirror.Env,
 			workingDir:        mirrorDir,
 			logDir:            logDir,

worker/rsync_provider.go

@@ -14,7 +14,9 @@ type rsyncConfig struct {
 	rsyncCmd                                     string
 	upstreamURL, username, password, excludeFile string
 	extraOptions                                 []string
+	globalOptions                                []string
 	overriddenOptions                            []string
+	useOverrideOnly                              bool
 	rsyncNeverTimeout                            bool
 	rsyncTimeoutValue                            int
 	rsyncEnv                                     map[string]string
@@ -75,25 +77,36 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) {
 		options = c.overriddenOptions
 	}
 
-	if !c.rsyncNeverTimeout {
-		timeo := 120
-		if c.rsyncTimeoutValue > 0 {
-			timeo = c.rsyncTimeoutValue
+	if c.useOverrideOnly == true {
+		if c.overriddenOptions == nil {
+			return nil, errors.New("rsync_override_only is set but no rsync_override provided")
+		}
+		// use overridden options only
+	} else {
+		if !c.rsyncNeverTimeout {
+			timeo := 120
+			if c.rsyncTimeoutValue > 0 {
+				timeo = c.rsyncTimeoutValue
+			}
+			options = append(options, fmt.Sprintf("--timeout=%d", timeo))
 		}
-		options = append(options, fmt.Sprintf("--timeout=%d", timeo))
-	}
 
-	if c.useIPv6 {
-		options = append(options, "-6")
-	} else if c.useIPv4 {
-		options = append(options, "-4")
-	}
+		if c.useIPv6 {
+			options = append(options, "-6")
+		} else if c.useIPv4 {
+			options = append(options, "-4")
+		}
 
-	if c.excludeFile != "" {
-		options = append(options, "--exclude-from", c.excludeFile)
-	}
-	if c.extraOptions != nil {
-		options = append(options, c.extraOptions...)
+		if c.excludeFile != "" {
+			options = append(options, "--exclude-from", c.excludeFile)
+		}
+
+		if c.globalOptions != nil {
+			options = append(options, c.globalOptions...)
+		}
+		if c.extraOptions != nil {
+			options = append(options, c.extraOptions...)
+		}
 	}
 	provider.options = options