add config examples of the worker (#118)

docs/zh_CN/workers.conf

@@ -1,3 +1,4 @@
+# /home/scripts in this example points to https://github.com/tuna/tunasync-scripts/
 
 [global]
 name = "mirror_worker"
@@ -22,52 +23,637 @@ listen_addr = "127.0.0.1"
 listen_port = 6000
 ssl_cert = ""
 ssl_key = ""
-
 [[mirrors]]
 name = "adobe-fonts"
 interval = 1440
 provider = "command"
 upstream = "https://github.com/adobe-fonts"
-#https://github.com/tuna/tunasync-scripts/blob/master/adobe-fonts.sh
 command = "/home/scripts/adobe-fonts.sh"
+size_pattern = "Total size is ([0-9\\.]+[KMGTP]?)"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+
+[[mirrors]]
+name = "AdoptOpenJDK"
+interval = 5760
+provider = "command"
+command = "/home/scripts/adoptopenjdk.py"
+upstream = "https://adoptopenjdk.jfrog.io/adoptopenjdk"
 docker_image = "tunathu/tunasync-scripts:latest"
 
+[[mirrors]]
+name = "alpine"
+provider = "rsync"
+upstream = "rsync://rsync.alpinelinux.org/alpine/"
+memory_limit = "256M"
+
 [[mirrors]]
 name = "anaconda"
 provider = "command"
 upstream = "https://repo.continuum.io/"
-#https://github.com/tuna/tunasync-scripts/blob/master/anaconda.py
-command = "/home/scripts/anaconda.py"
+command = "/home/scripts/anaconda.py --delete"
+size_pattern = "Total size is ([0-9\\.]+[KMGTP]?)"
+interval = 720
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "apache"
+provider = "rsync"
+upstream = "rsync://rsync.apache.org/apache-dist/"
+use_ipv4 = true
+rsync_options = [ "--delete-excluded" ]
+memory_limit = "256M"
+
+[[mirrors]]
+name = "armbian"
+provider = "two-stage-rsync"
+stage1_profile = "debian"
+upstream = "rsync://rsync.armbian.com/apt/"
+memory_limit = "256M"
+
+[[mirrors]]
+name = "armbian-releases"
+provider = "rsync"
+stage1_profile = "debian"
+upstream = "rsync://rsync.armbian.com/dl/"
+memory_limit = "256M"
+
+[[mirrors]]
+name = "bananian"
+provider = "command"
+upstream = "https://dl.bananian.org/"
+command = "/home/scripts/lftp.sh"
+interval = 1440
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "bioconductor"
+provider = "rsync"
+upstream = "master.bioconductor.org:./"
+rsync_options = [ "--rsh=ssh -i /root/id_rsa -o PasswordAuthentication=no -l sync" ]
+exclude_file = "/etc/excludes/bioconductor.txt"
+memory_limit = "256M"
+
+[[mirrors]]
+name = "blender"
+provider = "rsync"
+upstream = "rsync://mirrors.dotsrc.org/blender/"
+rsync_options = [ "--delete-excluded" ]
+exclude_file = "/etc/excludes/blender.txt"
+interval = 1440
+memory_limit = "256M"
+
+[[mirrors]]
+name = "chakra"
+provider = "rsync"
+upstream = "rsync://rsync.chakralinux.org/packages/"
+memory_limit = "256M"
+
+[[mirrors]]
+name = "chakra-releases"
+provider = "rsync"
+upstream = "rsync://rsync.chakralinux.org/releases/"
+memory_limit = "256M"
+
+[[mirrors]]
+name = "chef"
+interval = 1440
+provider = "command"
+upstream = "https://packages.chef.io/repos"
+command = "/home/scripts/chef.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "clickhouse"
+interval = 2880
+provider = "rsync"
+upstream = "rsync://repo.yandex.ru/yandexrepo/clickhouse/"
+exclude_file = "/etc/excludes/clickhouse.txt"
+memory_limit = "256M"
+
+[[mirrors]]
+name = "clojars"
+provider = "command"
+upstream = "s3://clojars-repo-production/"
+command = "/home/scripts/s3.sh"
+docker_image = "tunathu/ftpsync:latest"
+
+        [mirrors.env]
+        TUNASYNC_S3_ENDPOINT = "https://s3.dualstack.us-east-2.amazonaws.com"
+        #TUNASYNC_S3_ENDPOINT = "https://s3.us-east-2.amazonaws.com"
+        TUNASYNC_AWS_OPTIONS = "--delete --exclude index.html"
+
+[[mirrors]]
+name = "CPAN"
+provider = "rsync"
+upstream = "rsync://cpan-rsync.perl.org/CPAN/"
+memory_limit = "256M"
+
+[[mirrors]]
+name = "CRAN"
+provider = "rsync"
+upstream = "rsync://cran.r-project.org/CRAN/"
+rsync_options = [ "--delete-excluded" ]
+memory_limit = "256M"
+
+[[mirrors]]
+name = "CTAN"
+provider = "rsync"
+upstream = "rsync://mirrors.rit.edu/CTAN/"
+memory_limit = "256M"
+
+[[mirrors]]
+name = "dart-pub"
+provider = "command"
+upstream = "https://pub.dev/api"
+command = "/home/scripts/pub.sh"
+interval = 30
+docker_image = "tunathu/pub-mirror:latest"
+
+        [mirrors.env] 
+        MIRROR_BASE_URL = "https://mirrors.tuna.tsinghua.edu.cn/dart-pub"
+
+[[mirrors]]
+name = "debian"
+provider = "command"
+upstream = "rsync://mirrors.tuna.tsinghua.edu.cn/debian/"
+command = "/home/scripts/debian.sh sync:archive:debian"
+size_pattern = "Total size is ([0-9\\.]+[KMGTP]?)"
+docker_image = "tunathu/ftpsync"
+docker_volumes = [
+        "/etc/misc/ftpsync-debian.conf:/ftpsync/etc/ftpsync-debian.conf:ro",
+        "/log/ftpsync:/home/log/tunasync/ftpsync",
+]
+        [mirrors.env]
+        FTPSYNC_LOG_DIR = "/home/log/tunasync/ftpsync"
+
+[[mirrors]]
+name = "docker-ce"
+provider = "command"
+upstream = "https://download.docker.com/"
+command = "timeout 3h /home/scripts/docker-ce.py --workers 10 --fast-skip"
+interval = 1440
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "ELK"
+interval = 1440
+provider = "command"
+upstream = "https://packages.elastic.co"
+command = "/home/scripts/ELK.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
+docker_image = "tunathu/tunasync-scripts:latest"
+        # set environment varialbes
+        [mirrors.env]
+        WGET_OPTIONS = "-6"
+
+[[mirrors]]
+name = "elasticstack"
+interval = 1440
+provider = "command"
+upstream = "https://artifacts.elastic.co/"
+command = "/home/scripts/elastic.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "erlang-solutions"
+interval = 1440
+provider = "command"
+upstream = "https://packages.erlang-solutions.com"
+command = "/home/scripts/erlang.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "flutter"
+interval = 1440
+provider = "command"
+upstream = "https://storage.googleapis.com/flutter_infra/"
+command = "/home/scripts/flutter.sh"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "github-release"
+provider = "command"
+upstream = "https://api.github.com/repos/"
+command = "/home/scripts/github-release.py --workers 5"
+size_pattern = "Total size is ([0-9\\.]+[KMGTP]?)"
+interval = 720
+docker_image = "tunathu/tunasync-scripts:latest"
+  [mirrors.env]
+  GITHUB_TOKEN = "xxxxx"
+
+[[mirrors]]
+name = "gitlab-ce"
+interval = 1440
+provider = "command"
+upstream = "https://packages.gitlab.com/gitlab/gitlab-ce/"
+command = "/home/scripts/gitlab-ce.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "gitlab-ee"
+interval = 1440
+provider = "command"
+upstream = "https://packages.gitlab.com/gitlab/gitlab-ee/"
+command = "/home/scripts/gitlab-ce.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "gitlab-runner"
+interval = 1440
+provider = "command"
+upstream = "https://packages.gitlab.com/runner/gitlab-runner"
+command = "/home/scripts/gitlab-runner.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "grafana"
+interval = 1440
+provider = "command"
+upstream = "https://packages.grafana.com/oss"
+command = "/home/scripts/grafana.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "hackage"
+provider = "command"
+command = "/home/scripts/hackage.sh"
+upstream = "https://hackage.haskell.org/"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "homebrew-bottles"
+provider = "command"
+upstream = "https://homebrew.bintray.com"
+command = "/home/scripts/linuxbrew-bottles.sh"
+docker_image = "tunathu/homebrew-mirror"
+        # set environment varialbes
+        [mirrors.env]
+        HOMEBREW_REPO = "https://neomirrors.tuna.tsinghua.edu.cn/git/homebrew"
+
+[[mirrors]]
+name = "influxdata"
 interval = 1440
+provider = "command"
+upstream = "https://repos.influxdata.com"
+command = "/home/scripts/influxdata.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
 docker_image = "tunathu/tunasync-scripts:latest"
 
 [[mirrors]]
-name = "gnu"
+name = "kali"
+provider = "two-stage-rsync"
+stage1_profile = "debian"
+upstream = "rsync://ftp.nluug.nl/kali/"
+rsync_options = [ "--delete-excluded" ] # delete .~tmp~ folders
+memory_limit = "256M"
+
+[[mirrors]]
+name = "kali-images"
+provider = "rsync"
+upstream = "rsync://ftp.nluug.nl/kali-images/"
+rsync_options = [ "--delete-excluded" ] # delete .~tmp~ folders
+memory_limit = "256M"
+
+[[mirrors]]
+name = "KaOS"
+provider = "rsync"
+upstream = "rsync://kaosx.tk/kaos/"
+rsync_options = [ "--delete-excluded" ]
+memory_limit = "256M"
+
+[[mirrors]]
+name = "kernel"
 provider = "rsync"
-upstream = "rsync://mirrors.ocf.berkeley.edu/gnu/"
+upstream = "rsync://rsync.kernel.org/pub/linux/kernel/"
+rsync_options = [ "--delete-excluded" ]
+memory_limit = "256M"
+
+[[mirrors]]
+name = "kicad"
+provider = "command"
+upstream = "s3://kicad-downloads/"
+command = "/home/scripts/s3.sh"
+docker_image = "tunathu/ftpsync:latest"
+        [mirrors.env]
+        TUNASYNC_S3_ENDPOINT = "https://s3.cern.ch"
+        TUNASYNC_AWS_OPTIONS = "--delete --exclude index.html"
+
+[[mirrors]]
+name = "kodi"
+provider = "rsync"
+upstream = "rsync://mirror.yandex.ru/mirrors/xbmc/"
+rsync_options = [ "--delete-excluded" ]
+memory_limit = "256M"
+use_ipv6 = true
+
+[[mirrors]]
+name = "kubernetes"
+interval = 2880
+provider = "command"
+upstream = "http://packages.cloud.google.com"
+command = "/home/scripts/kubernetes.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "linuxbrew-bottles"
+provider = "command"
+upstream = "https://linuxbrew.bintray.com"
+command = "/home/scripts/linuxbrew-bottles.sh"
+docker_image = "tunathu/homebrew-mirror"
+        # set environment varialbes
+        [mirrors.env]
+        RUN_LINUXBREW = "true"
+        HOMEBREW_REPO = "https://neomirrors.tuna.tsinghua.edu.cn/git/homebrew"
+
+[[mirrors]]
+name = "linuxmint"
+provider = "two-stage-rsync"
+stage1_profile = "debian"
+upstream = "rsync://mirrors.kernel.org/linuxmint-packages/"
 rsync_options = [ "--delete-excluded" ]
 memory_limit = "256M"
 
+[[mirrors]]
+name = "lxc-images"
+provider = "command"
+upstream = "https://us.images.linuxcontainers.org/"
+command = "/home/scripts/lxc-images.sh"
+docker_image = "tunathu/tunasync-scripts:latest"
+interval = 720
+
+[[mirrors]]
+name = "lyx"
+provider = "command"
+upstream = "ftp://ftp.lyx.org/pub/lyx/"
+command = "/home/scripts/lftp.sh"
+docker_image = "tunathu/tunasync-scripts:latest"
+        [mirrors.env]
+        TUNASYNC_LFTP_OPTIONS = "--only-newer"
+
+[[mirrors]]
+name = "mongodb"
+interval = 1440
+provider = "command"
+upstream = "https://repo.mongodb.org"
+command = "/home/scripts/mongodb.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "msys2"
+provider = "command"
+upstream = "http://repo.msys2.org/"
+command = "/home/scripts/lftp.sh"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "mysql"
+interval = 30
+provider = "command"
+upstream = "https://repo.mysql.com"
+command = "/home/scripts/mysql.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
+docker_image = "tunathu/tunasync-scripts:latest"
+        # set environment varialbes
+        [mirrors.env]
+        USE_IPV6 = "1"
+
+[[mirrors]]
+name = "nix"
+interval = 1440
+provider = "command"
+upstream = "s3://nix-releases/nix/"
+command = "/home/scripts/nix.sh"
+docker_image = "tunathu/tunasync-scripts:latest"
+        [mirrors.env]
+        MIRROR_BASE_URL = 'https://mirrors.tuna.tsinghua.edu.cn/nix/'
+
+[[mirrors]]
+name = "nix-channels"
+interval = 300
+provider = "command"
+upstream = "https://nixos.org/channels"
+command = "timeout 20h /home/scripts/nix-channels.py"
+docker_image = "tunathu/nix-channels:latest"
+docker_options = [
+  "--cpus", "20",
+  ]
+
+[[mirrors]]
+name = "nodesource"
+provider = "command"
+upstream = "https://deb.nodesource.com/"
+command = "/home/scripts/nodesource.sh"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "openresty"
+provider = "command"
+upstream = "https://openresty.org/package/"
+command = "/home/scripts/lftp.sh"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+        [mirrors.env]
+        TUNASYNC_LFTP_OPTIONS = "--only-newer"
+
+[[mirrors]]
+name = "packagist"
+provider = "command"
+upstream = "http://packagist.org/"
+command = "/home/scripts/packagist.sh"
+interval = 1440
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "proxmox"
+interval = 1440
+provider = "command"
+upstream = "http://download.proxmox.com"
+command = "/home/scripts/proxmox.sh"
+docker_image = "tunathu/tunasync-scripts:latest"
+
 [[mirrors]]
 name = "pypi"
 provider = "command"
 upstream = "https://pypi.python.org/"
-#https://github.com/tuna/tunasync-scripts/blob/master/pypi.sh
 command = "/home/scripts/pypi.sh"
 docker_image = "tunathu/tunasync-scripts:latest"
 interval = 5
+
+[[mirrors]]
+name = "qt"
+provider = "rsync"
+upstream = "rsync://master.qt-project.org/qt-all/"
+exclude_file = "/etc/excludes/qt.txt"
+rsync_options = [ "--delete-excluded" ]
+memory_limit = "256M"
+
+[[mirrors]]
+name = "raspberrypi"
+provider = "two-stage-rsync"
+stage1_profile = "debian"
+upstream = "rsync://apt-repo.raspberrypi.org/archive/debian/"
+memory_limit = "256M"
+
+[[mirrors]]
+name = "raspbian-images"
+interval = 5760
+provider = "command"
+upstream = "https://downloads.raspberrypi.org/"
+command = "/home/scripts/lftp.sh"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+        [mirrors.env]
+        TUNASYNC_LFTP_OPTIONS = "-x ^icons/$ -c --only-missing -v --no-perms"
+
+[[mirrors]]
+name = "raspbian"
+provider = "two-stage-rsync"
+stage1_profile = "debian"
+upstream = "rsync://archive.raspbian.org/archive/"
+rsync_options = [ "--delete-excluded" ] # delete .~tmp~ folders
+memory_limit = "256M"
+
+
+[[mirrors]]
+name = "redhat"
+provider = "rsync"
+upstream = "rsync://ftp.redhat.com/redhat/"
+rsync_options = [ "--delete-excluded" ]
+memory_limit = "256M"
+exclude_file = "/etc/excludes/redhat.txt"
+interval = 1440
+
+        [mirrors.env]
+        RSYNC_PROXY="127.0.0.1:8123"
+
+
+[[mirrors]]
+name = "remi"
+interval = 1440
+provider = "command"
+upstream = "rsync://rpms.remirepo.net"
+command = "/home/scripts/remi.sh"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "repo-ck"
+provider = "command"
+upstream = "http://repo-ck.com"
+command = "/home/scripts/repo-ck.sh"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "ros"
+provider = "rsync"
+upstream = "rsync://mirror.umd.edu/packages.ros.org/ros/"
+memory_limit = "256M"
+
+[[mirrors]]
+name = "ros2"
+interval = 1440
+provider = "command"
+upstream = "http://packages.ros.org/ros2"
+command = "/home/scripts/ros2.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "rubygems"
+provider = "command"
+upstream = "https://rubygems.org"
+command = "/home/scripts/rubygems.sh"
+docker_image = "tunathu/rubygems-mirror"
+interval = 60
         # set environment varialbes
         [mirrors.env]
         INIT = "0"
 
+[[mirrors]]
+name = "rudder"
+interval = 2880
+provider = "command"
+upstream = "https://repository.rudder.io"
+command = "/home/scripts/rudder.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
+docker_image = "tunathu/tunasync-scripts:latest"
+
 
 [[mirrors]]
-name = "debian"
-interval = 720
+name = "rustup"
+provider = "command"
+upstream = "https://rustup.rs/"
+command = "/home/scripts/rustup.sh"
+interval = 1440
+docker_image = "tunathu/rustup-mirror:latest"
+docker_volumes = [
+]
+docker_options = [
+]
+        [mirrors.env] 
+        MIRROR_BASE_URL = "https://mirrors.tuna.tsinghua.edu.cn/rustup"
+
+[[mirrors]]
+name = "saltstack"
+interval = 1440 # required on http://repo.saltstack.com/#mirror
+provider = "command"
+upstream = "s3://s3/"
+command = "/home/scripts/s3.sh"
+docker_image = "tunathu/ftpsync:latest"
+
+        [mirrors.env]
+        TUNASYNC_S3_ENDPOINT = "https://s3.repo.saltstack.com"
+        TUNASYNC_AWS_OPTIONS = "--delete --exact-timestamps"
+
+[[mirrors]]
+name = "solus"
 provider = "rsync"
-upstream = "rsync://mirrors.tuna.tsinghua.edu.cn/debian/"
+upstream = "rsync://mirrors.rit.edu/solus/"
+rsync_options = [ "--exclude", "/shannon", "--exclude", "/unstable" ]
 memory_limit = "256M"
 
+
+[[mirrors]]
+name = "stackage"
+provider = "command"
+command = "/home/scripts/stackage.py"
+upstream = "https://www.stackage.org/"
+docker_image = "tunathu/tunasync-scripts:latest"
+        # set environment varialbes
+        [mirrors.env]
+        GIT_COMMITTER_NAME = "TUNA mirrors"
+        GIT_COMMITTER_EMAIL = "mirrors@tuna.tsinghua.edu.cn"
+
+
+[[mirrors]]
+name = "steamos"
+interval = 1440
+provider = "command"
+upstream = "http://repo.steampowered.com"
+command = "/home/scripts/lftp.sh"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+        [mirrors.env]
+    TUNASYNC_LFTP_OPTIONS = "--only-newer --exclude icons/ "
+
+[[mirrors]]
+name = "termux"
+interval = 1440
+provider = "command"
+upstream = "https://dl.bintray.com/termux/termux-packages-24/"
+command = "/home/scripts/termux.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
+docker_image = "tunathu/tunasync-scripts:latest"
+
 [[mirrors]]
 name = "ubuntu"
 provider = "two-stage-rsync"
@@ -76,4 +662,39 @@ upstream = "rsync://archive.ubuntu.com/ubuntu/"
 rsync_options = [ "--delete-excluded" ]
 memory_limit = "256M"
 
-# vim: ft=toml
+[[mirrors]]
+name = "ubuntu-ports"
+provider = "two-stage-rsync"
+stage1_profile = "debian"
+upstream = "rsync://ports.ubuntu.com/ubuntu-ports/"
+rsync_options = [ "--delete-excluded" ]
+exclude_file = "/etc/excludes/ubuntu-ports-exclude.txt"
+memory_limit = "256M"
+
+[[mirrors]]
+name = "virtualbox"
+interval = 1440
+provider = "command"
+upstream = "http://download.virtualbox.org/virtualbox"
+command = "/home/scripts/virtualbox.sh"
+size_pattern = "size-sum: ([0-9\\.]+[KMGTP])"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+[[mirrors]]
+name = "winehq"
+provider = "command"
+upstream = "ftp://ftp.winehq.org/pub/"
+command = "/home/scripts/lftp.sh"
+docker_image = "tunathu/tunasync-scripts:latest"
+
+        [mirrors.env]
+        TUNASYNC_LFTP_OPTIONS = "-x wine-builds.old/ -x /\\..+"
+
+[[mirrors]]
+name = "zabbix"
+provider = "rsync"
+upstream = "rsync://repo.zabbix.com/mirror/"
+rsync_options = [ "--delete-excluded", "--chmod=o+r,Do+x,Fa-x" ]
+memory_limit = "256M"
+
+# vim: ft=toml