Merge pull request #51 from tuna/dev

Dev

.testandcover.bash

@@ -1,6 +1,4 @@
 #!/bin/bash
-
-
 function die() {
   echo $*
   exit 1
@@ -8,7 +6,7 @@ function die() {
 
 export GOPATH=`pwd`:$GOPATH
 
-make
+make travis
 
 # Initialize profile.cov
 echo "mode: count" > profile.cov
@@ -19,10 +17,9 @@ ERROR=""
 # Test each package and append coverage profile info to profile.cov
 for pkg in `cat .testpackages.txt`
 do
-    #$HOME/gopath/bin/
-    go test -v -covermode=count -coverprofile=profile_tmp.cov $pkg || ERROR="Error testing $pkg"
-
-    [ -f profile_tmp.cov ] && {
+	go test -v -covermode=count -coverprofile=profile_tmp.cov $pkg || ERROR="Error testing $pkg"
+	
+	[ -f profile_tmp.cov ] && {
 		tail -n +2 profile_tmp.cov >> profile.cov || die "Unable to append coverage for $pkg"
 	}
 done

.travis.yml

@@ -19,3 +19,17 @@ script:
 
 after_success:
     - goveralls -coverprofile=profile.cov -service=travis-ci
+
+before_deploy: "echo 'ready to deploy?'"
+
+deploy:
+    provider: releases
+    file:
+        - "build/tunasync-linux-bin.tar.gz"
+    api_key:
+        secure: "F9kaVaR1mxEh2+EL9Nm8GZmbVY98pXCJA0LGDNrq1C2vU61AUNOeX6yI1mMklHNZPLBqoFDvGN1M5HnJ+xWCFH+KnJgLD2GVIAcAxFNpcNWQe8XKE5heklNsIQNQfuh/rJKM6YzeDB9G5RN4Y76iL4WIAXhNnMm48W6jLnWhf70="
+    skip_cleanup: true
+    overwrite: true
+    on:
+        tags: true
+        all_branches: true

Makefile

@@ -2,6 +2,8 @@ LDFLAGS="-X main.buildstamp=`date -u '+%s'` -X main.githash=`git rev-parse HEAD`
 
 all: get tunasync tunasynctl
 
+travis: get tunasync tunasynctl travis-package
+
 get: 
 	go get ./cmd/tunasync
 	go get ./cmd/tunasynctl
@@ -14,3 +16,6 @@ tunasync: build
 
 tunasynctl: build
 	go build -o build/tunasynctl -ldflags ${LDFLAGS} github.com/tuna/tunasync/cmd/tunasynctl
+
+travis-package: tunasync tunasynctl
+	tar zcf build/tunasync-linux-bin.tar.gz -C build tunasync tunasynctl

README.md

@@ -10,6 +10,9 @@ tunasync
 
 - [中文文档](https://github.com/tuna/tunasync/blob/master/docs/zh_CN/get_started.md)
 
+## Download
+
+Pre-built binary for Linux x86_64 is available at [Github releases](https://github.com/tuna/tunasync/releases/latest).
 
 ## Design
 

cmd/tunasync/tunasync.go

@@ -8,10 +8,10 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/codegangsta/cli"
 	"github.com/gin-gonic/gin"
 	"github.com/pkg/profile"
 	"gopkg.in/op/go-logging.v1"
+	"gopkg.in/urfave/cli.v1"
 
 	tunasync "github.com/tuna/tunasync/internal"
 	"github.com/tuna/tunasync/manager"
@@ -25,7 +25,7 @@ var (
 
 var logger = logging.MustGetLogger("tunasync")
 
-func startManager(c *cli.Context) {
+func startManager(c *cli.Context) error {
 	tunasync.InitLogger(c.Bool("verbose"), c.Bool("debug"), c.Bool("with-systemd"))
 
 	cfg, err := manager.LoadConfig(c.String("config"), c)
@@ -45,9 +45,10 @@ func startManager(c *cli.Context) {
 
 	logger.Info("Run tunasync manager server.")
 	m.Run()
+	return nil
 }
 
-func startWorker(c *cli.Context) {
+func startWorker(c *cli.Context) error {
 	tunasync.InitLogger(c.Bool("verbose"), c.Bool("debug"), c.Bool("with-systemd"))
 	if !c.Bool("debug") {
 		gin.SetMode(gin.ReleaseMode)
@@ -103,6 +104,7 @@ func startWorker(c *cli.Context) {
 
 	logger.Info("Run tunasync worker.")
 	w.Run()
+	return nil
 }
 
 func main() {
@@ -129,6 +131,8 @@ func main() {
 	}
 
 	app := cli.NewApp()
+	app.Name = "tunasync"
+	app.Usage = "tunasync mirror job management tool"
 	app.EnableBashCompletion = true
 	app.Version = "0.1"
 	app.Commands = []cli.Command{
@@ -158,10 +162,6 @@ func main() {
 					Name:  "key",
 					Usage: "Use SSL key from `FILE`",
 				},
-				cli.StringFlag{
-					Name:  "status-file",
-					Usage: "Write status file to `FILE`",
-				},
 				cli.StringFlag{
 					Name:  "db-file",
 					Usage: "Use `FILE` as the database file",
@@ -205,7 +205,7 @@ func main() {
 				},
 				cli.BoolFlag{
 					Name:  "debug",
-					Usage: "Run manager in debug mode",
+					Usage: "Run worker in debug mode",
 				},
 				cli.BoolFlag{
 					Name:  "with-systemd",

cmd/tunasynctl/tunasynctl.go

@@ -11,8 +11,8 @@ import (
 	"time"
 
 	"github.com/BurntSushi/toml"
-	"github.com/codegangsta/cli"
 	"gopkg.in/op/go-logging.v1"
+	"gopkg.in/urfave/cli.v1"
 
 	tunasync "github.com/tuna/tunasync/internal"
 )
@@ -323,6 +323,8 @@ func main() {
 	app := cli.NewApp()
 	app.EnableBashCompletion = true
 	app.Version = "0.1"
+	app.Name = "tunasynctl"
+	app.Usage = "control client for tunasync manager"
 
 	commonFlags := []cli.Flag{
 		cli.StringFlag{

docs/zh_CN/get_started.md

@@ -15,7 +15,7 @@ date: 2016-10-31 00:50:00
 
 ### 二进制包
 
-TODO
+到 [Github Releases](https://github.com/tuna/tunasync/releases/latest) 下载 `tunasync-linux-bin.tar.gz` 即可。
 
 ### 自行编译
 
@@ -57,8 +57,11 @@ listen_port = 6000
 ssl_cert = ""
 ssl_key = ""
 
-[include]
-include_mirrors = "mirrors/*.conf"
+[[mirrors]]
+name = "elvish"
+provider = "rsync"
+upstream = "rsync://rsync.elvish.io/elvish/"
+use_ipv6 = false
 ```
 
 `~/tunasync_demo/manager.conf`:
@@ -78,20 +81,6 @@ db_file = "/tmp/tunasync/manager.db"
 ca_cert = ""
 ```
 
-### 镜像脚本
-
-```
-$ mkdir ~/tunasync_demo/mirrors
-$ cat > ~/tunasync_demo/mirrors/elvish.conf < EOF
-
-[[mirrors]]
-name = "elvish"
-provider = "rsync"
-upstream = "rsync://rsync.elvish.io/elvish/"
-use_ipv6 = false
-EOF
-```
-
 ### 运行
 
 ```

manager/config.go

@@ -2,7 +2,7 @@ package manager
 
 import (
 	"github.com/BurntSushi/toml"
-	"github.com/codegangsta/cli"
+	"gopkg.in/urfave/cli.v1"
 )
 
 // A Config is the top-level toml-serializaible config struct

manager/config_test.go

@@ -8,8 +8,8 @@ import (
 	"testing"
 
 	"github.com/BurntSushi/toml"
-	"github.com/codegangsta/cli"
 	. "github.com/smartystreets/goconvey/convey"
+	"gopkg.in/urfave/cli.v1"
 )
 
 func TestConfig(t *testing.T) {

scripts/anaconda.sh

@@ -1,96 +0,0 @@
-#!/bin/bash
-# requires: wget, lftp, jq
-#
-
-set -e
-set -o pipefail
-
-CONDA_REPO_BASE=${CONDA_REPO_BASE:-"http://repo.continuum.io"}
-LOCAL_DIR_BASE="${TUNASYNC_WORKING_DIR}/pkgs"
-TMP_DIR=$(mktemp -d)
-
-CONDA_REPOS=("free" "r" "mro" "pro")
-CONDA_ARCHES=("linux-64" "linux-32" "linux-armv6l" "linux-armv7l" "linux-ppc64le" "osx-64" "osx-32" "win-64" "win-32")
-
-function check-and-download() {
-	remote_file=$1
-	local_file=$2
-	wget -q --spider ${remote_file}
-	if [ $? -eq 0 ]; then
-		echo "downloading ${remote_file}"
-		wget -q -N -O ${local_file} ${remote_file}
-		return
-	fi
-	return 1
-}
-
-function cleanup () {
-	echo "cleaning up"
-	[ -d ${TMP_DIR} ] && {
-		[ -f ${TMP_DIR}/repodata.json ] && rm ${TMP_DIR}/repodata.json
-		[ -f ${TMP_DIR}/repodata.json.bz2 ] && rm ${TMP_DIR}/repodata.json.bz2
-		rmdir ${TMP_DIR}
-	}
-}
-
-trap cleanup EXIT
-
-echo ${TMP_DIR}
-
-for repo in ${CONDA_REPOS[@]}; do
-	for arch in ${CONDA_ARCHES[@]}; do
-		PKG_REPO_BASE="${CONDA_REPO_BASE}/pkgs/$repo/$arch"
-		repodata_url="${PKG_REPO_BASE}/repodata.json"
-		bz2_repodata_url="${PKG_REPO_BASE}/repodata.json.bz2"
-		LOCAL_DIR="${LOCAL_DIR_BASE}/$repo/$arch"
-		[ ! -d ${LOCAL_DIR} ] && mkdir -p ${LOCAL_DIR}
-		tmp_repodata="${TMP_DIR}/repodata.json"
-		tmp_bz2_repodata="${TMP_DIR}/repodata.json.bz2"
-
-		check-and-download ${repodata_url} ${tmp_repodata}
-		check-and-download ${bz2_repodata_url} ${tmp_bz2_repodata}
-
-		jq_cmd='.packages | to_entries[] | [.key, .value.size, .value.md5] | map(tostring) | join(" ")'
-		bzip2 -c -d ${tmp_bz2_repodata} | jq -r "${jq_cmd}" | while read line;
-		do
-			read -a tokens <<< $line
-			pkgfile=${tokens[0]}
-			pkgsize=${tokens[1]}
-			pkgmd5=${tokens[2]}
-			
-			pkg_url="${PKG_REPO_BASE}/${pkgfile}"
-			dest_file="${LOCAL_DIR}/${pkgfile}"
-			
-			declare downloaded=false
-			if [ -f ${dest_file} ]; then
-				rsize=`stat -c "%s" ${dest_file}`
-				if [ ${rsize} -eq ${pkgsize} ]; then
-					downloaded=true
-					echo "Skipping ${pkgfile}, size ${pkgsize}"
-				fi
-			fi
-			while [ $downloaded != true ]; do
-				echo "downloading ${pkg_url}"
-				wget -q -O ${dest_file} ${pkg_url} && {
-					# two space for md5sum check format
-					echo "${pkgmd5}  ${dest_file}" | md5sum -c - && downloaded=true 
-				}
-			done
-		done
-		
-		mv -f "${TMP_DIR}/repodata.json" "${LOCAL_DIR}/repodata.json"
-		mv -f "${TMP_DIR}/repodata.json.bz2" "${LOCAL_DIR}/repodata.json.bz2"
-	done
-done
-
-function sync_installer() {
-	repo_url="$1"
-	repo_dir="$2"
-
-	[ ! -d "$repo_dir" ] && mkdir -p "$repo_dir"
-	cd $repo_dir
-	lftp "${repo_url}/" -e "mirror --verbose -P 5; bye"
-}
-
-sync_installer "${CONDA_REPO_BASE}/archive/" "${TUNASYNC_WORKING_DIR}/archive/"
-sync_installer "${CONDA_REPO_BASE}/miniconda/" "${TUNASYNC_WORKING_DIR}/miniconda/"

scripts/aosp.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-REPO=${REPO:-"/usr/local/bin/repo"}
-
-function repo_init() {
-	mkdir -p $TUNASYNC_WORKING_DIR
-	cd $TUNASYNC_WORKING_DIR
-	$REPO init -u https://android.googlesource.com/mirror/manifest --mirror
-}
-
-function repo_sync() {
-	cd $TUNASYNC_WORKING_DIR
-	$REPO sync -f
-}
-
-if [ ! -d "$TUNASYNC_WORKING_DIR/git-repo.git" ]; then
-	echo "Initializing AOSP mirror"
-	repo_init
-fi
-
-repo_sync

scripts/docker.sh

@@ -1,59 +0,0 @@
-#!/bin/bash
-# reqires: wget, yum-utils
-
-set -e
-set -o pipefail
-
-_here=`dirname $(realpath $0)`
-. ${_here}/helpers/apt-download
-APT_VERSIONS=("debian-wheezy" "debian-jessie" "ubuntu-precise" "ubuntu-trusty" "ubuntu-xenial")
-
-BASE_PATH="${TUNASYNC_WORKING_DIR}"
-APT_PATH="${BASE_PATH}/apt/repo"
-YUM_PATH="${BASE_PATH}/yum/repo"
-
-mkdir -p ${APT_PATH} ${YUM_PATH}
-
-wget -q -N -O ${BASE_PATH}/yum/gpg https://yum.dockerproject.org/gpg
-wget -q -N -O ${BASE_PATH}/apt/gpg https://apt.dockerproject.org/gpg
-
-# YUM mirror
-cache_dir="/tmp/yum-docker-cache/"
-cfg="/tmp/docker-yum.conf"
-cat <<EOF > ${cfg}
-[main]
-keepcache=0
-
-[centos6]
-name=Docker Repository
-baseurl=https://yum.dockerproject.org/repo/main/centos/6
-enabled=1
-gpgcheck=0
-gpgkey=https://yum.dockerproject.org/gpg
-sslverify=0
-
-[centos7]
-name=Docker Repository
-baseurl=https://yum.dockerproject.org/repo/main/centos/7
-enabled=1
-gpgcheck=0
-gpgkey=https://yum.dockerproject.org/gpg
-sslverify=0
-EOF
-
-[ ! -d ${YUM_PATH}/centos6 ] && mkdir -p ${YUM_PATH}/centos6
-[ ! -d ${YUM_PATH}/centos7 ] && mkdir -p ${YUM_PATH}/centos7
-reposync -c $cfg -d -p ${YUM_PATH}  -e $cache_dir
-createrepo --update -v -c $cache_dir -o ${YUM_PATH}/centos6 ${YUM_PATH}/centos7
-createrepo --update -v -c $cache_dir -o ${YUM_PATH}/centos6 ${YUM_PATH}/centos7
-rm $cfg
-
-# APT mirror 
-base_url="http://apt.dockerproject.org/repo"
-for version in ${APT_VERSIONS[@]}; do
-	apt-download-binary ${base_url} "$version" "main" "amd64" "${APT_PATH}" || true
-	apt-download-binary ${base_url} "$version" "main" "i386" "${APT_PATH}" || true
-done
-
-# sync_docker "http://apt.dockerproject.org/" "${TUNASYNC_WORKING_DIR}/apt"
-# sync_docker "http://yum.dockerproject.org/" "${TUNASYNC_WORKING_DIR}/yum"

scripts/gen_debian_exclude.py

@@ -1,13 +0,0 @@
-#!/usr/bin/env python
-
-ARCH_EXCLUDE = ['armel', 'alpha', 'hurd-i386', 'ia64', 'kfreebsd-amd64', 'kfreebsd-i386', 'mips', 'powerpc', 'ppc64el',  's390', 's390x', 'sparc']
-
-CONTENT_EXCLUDE = ['binary-{arch}', 'installer-{arch}', 'Contents-{arch}.gz', 'Contents-udeb-{arch}.gz', 'Contents-{arch}.diff', 'arch-{arch}.files', 'arch-{arch}.list.gz', '*_{arch}.deb', '*_{arch}.udeb', '*_{arch}.changes']
-
-with open("debian-exclude.txt", 'wb') as f:
-	f.write(".~tmp~/\n")
-	f.write(".*\n")
-	for arch in ARCH_EXCLUDE:
-		for content in CONTENT_EXCLUDE:
-			f.write(content.format(arch=arch))
-			f.write('\n')

scripts/gen_kali_exclude.py

@@ -1,13 +0,0 @@
-#!/usr/bin/env python
-
-ARCH_EXCLUDE = ['armel', 'armhf']
-
-CONTENT_EXCLUDE = ['binary-{arch}', 'installer-{arch}', 'Contents-{arch}.gz', 'Contents-udeb-{arch}.gz', 'Contents-{arch}.diff', 'arch-{arch}.files', 'arch-{arch}.list.gz', '*_{arch}.deb', '*_{arch}.udeb', '*_{arch}.changes']
-
-with open("kali-exclude.txt", 'wb') as f:
-	f.write(".~tmp~/\n")
-	f.write(".*\n")
-	for arch in ARCH_EXCLUDE:
-		for content in CONTENT_EXCLUDE:
-			f.write(content.format(arch=arch))
-			f.write('\n')

scripts/gen_ubuntu_ports_exclude.py

@@ -1,13 +0,0 @@
-#!/usr/bin/env python
-
-ARCH_EXCLUDE = ['powerpc', 'ppc64el', 'ia64', 'sparc', 'armel']
-
-CONTENT_EXCLUDE = ['binary-{arch}', 'installer-{arch}', 'Contents-{arch}.gz', 'Contents-udeb-{arch}.gz', 'Contents-{arch}.diff', 'arch-{arch}.files', 'arch-{arch}.list.gz', '*_{arch}.deb', '*_{arch}.udeb', '*_{arch}.changes']
-
-with open("ubuntu-ports-exclude.txt", 'wb') as f:
-	f.write(".~tmp~/\n")
-	f.write(".*\n")
-	for arch in ARCH_EXCLUDE:
-		for content in CONTENT_EXCLUDE:
-			f.write(content.format(arch=arch))
-			f.write('\n')

scripts/gitlab-ce.sh

@@ -1,65 +0,0 @@
-#!/bin/bash
-set -e
-
-_here=`dirname $(realpath $0)`
-. ${_here}/helpers/apt-download
-
-[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1)
-
-BASE_PATH="${TUNASYNC_WORKING_DIR}"
-
-YUM_PATH="${BASE_PATH}/yum"
-
-UBUNTU_VERSIONS=("trusty" "wily")
-DEBIAN_VERSIONS=("wheezy" "jessie" "stretch")
-UBUNTU_PATH="${BASE_PATH}/ubuntu/"
-DEBIAN_PATH="${BASE_PATH}/debian/"
-
-mkdir -p $UBUNTU_PATH $DEBIAN_PATH $YUM_PATH
-
-cache_dir="/tmp/yum-gitlab-ce-cache/"
-cfg="/tmp/gitlab-ce-yum.conf"
-cat <<EOF > ${cfg}
-[main]
-keepcache=0
-
-[el6]
-name=el6
-baseurl=https://packages.gitlab.com/gitlab/gitlab-ce/el/6/x86_64
-repo_gpgcheck=0
-gpgcheck=0
-enabled=1
-gpgkey=https://packages.gitlab.com/gpg.key
-sslverify=0
-
-[el7]
-name=el7
-baseurl=https://packages.gitlab.com/gitlab/gitlab-ce/el/7/x86_64
-repo_gpgcheck=0
-gpgcheck=0
-enabled=1
-gpgkey=https://packages.gitlab.com/gpg.key
-sslverify=0
-EOF
-
-reposync -c $cfg -d -p ${YUM_PATH}  -e $cache_dir
-createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el6 ${YUM_PATH}/el6
-createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el7 ${YUM_PATH}/el7
-rm $cfg
-
-base_url="https://packages.gitlab.com/gitlab/gitlab-ce/ubuntu"
-for version in ${UBUNTU_VERSIONS[@]}; do
-	apt-download-binary ${base_url} "$version" "main" "amd64" "${UBUNTU_PATH}" || true
-	apt-download-binary ${base_url} "$version" "main" "i386" "${UBUNTU_PATH}" || true
-done
-echo "Ubuntu finished"
-
-base_url="https://packages.gitlab.com/gitlab/gitlab-ce/debian"
-for version in ${DEBIAN_VERSIONS[@]}; do
-	apt-download-binary ${base_url} "$version" "main" "amd64" "${DEBIAN_PATH}" || true
-	apt-download-binary ${base_url} "$version" "main" "i386" "${DEBIAN_PATH}" || true
-done
-echo "Debian finished"
-
-
-# vim: ts=4 sts=4 sw=4

scripts/gitlab-ci-multi-runner.sh

@@ -1,69 +0,0 @@
-#!/bin/bash
-# reqires: wget, yum-utils
-set -e
-set -o pipefail
-
-_here=`dirname $(realpath $0)`
-. ${_here}/helpers/apt-download
-
-[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1)
-
-BASE_PATH="${TUNASYNC_WORKING_DIR}"
-
-YUM_PATH="${BASE_PATH}/yum"
-
-UBUNTU_VERSIONS=("trusty" "xenial")
-DEBIAN_VERSIONS=("wheezy" "jessie" "stretch")
-UBUNTU_PATH="${BASE_PATH}/ubuntu/"
-DEBIAN_PATH="${BASE_PATH}/debian/"
-
-mkdir -p $UBUNTU_PATH $DEBIAN_PATH $YUM_PATH
-
-cache_dir="/tmp/yum-gitlab-runner-cache/"
-cfg="/tmp/gitlab-runner-yum.conf"
-cat <<EOF > ${cfg}
-[main]
-keepcache=0
-
-[el6]
-name=gitlab-ci-multi-runner-el6
-baseurl=https://packages.gitlab.com/runner/gitlab-ci-multi-runner/el/6/x86_64
-repo_gpgcheck=0
-gpgcheck=0
-enabled=1
-gpgkey=https://packages.gitlab.com/gpg.key
-sslverify=0
-
-[el7]
-name=gitlab-ci-multi-runner-el7
-baseurl=https://packages.gitlab.com/runner/gitlab-ci-multi-runner/el/7/x86_64
-repo_gpgcheck=0
-gpgcheck=0
-enabled=1
-gpgkey=https://packages.gitlab.com/gpg.key
-sslverify=0
-EOF
-
-reposync -c $cfg -d -p ${YUM_PATH}  -e $cache_dir
-[ ! -d ${YUM_PATH}/el6 ] && mkdir -p ${YUM_PATH}/el6
-[ ! -d ${YUM_PATH}/el7 ] && mkdir -p ${YUM_PATH}/el7
-createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el6 ${YUM_PATH}/el6
-createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el7 ${YUM_PATH}/el7
-rm $cfg
-
-base_url="https://packages.gitlab.com/runner/gitlab-ci-multi-runner/ubuntu"
-for version in ${UBUNTU_VERSIONS[@]}; do
-	apt-download-binary ${base_url} "$version" "main" "amd64" "${UBUNTU_PATH}" || true
-	apt-download-binary ${base_url} "$version" "main" "i386" "${UBUNTU_PATH}" || true
-done
-echo "Ubuntu finished"
-
-base_url="https://packages.gitlab.com/runner/gitlab-ci-multi-runner/debian"
-for version in ${DEBIAN_VERSIONS[@]}; do
-	apt-download-binary ${base_url} "$version" "main" "amd64" "${DEBIAN_PATH}" || true
-	apt-download-binary ${base_url} "$version" "main" "i386" "${DEBIAN_PATH}" || true
-done
-echo "Debian finished"
-
-
-# vim: ts=4 sts=4 sw=4

scripts/hackage.sh

@@ -1,92 +0,0 @@
-#!/bin/bash
-set -e
-
-function remove_broken() {
-	interval=$1
-	interval_file="/tmp/hackage_lastcheck"
-	now=`date +%s`
-
-	if [[ -f ${interval_file} ]]; then
-		lastcheck=`cat ${interval_file}`
-		between=$(echo "${now}-${lastcheck}" | bc)
-		[[ $between -lt $interval ]] && echo "skip checking"; return 0
-	fi
-	echo "start checking"
-
-	mkdir -p "${TUNASYNC_WORKING_DIR}/package"
-	cd "${TUNASYNC_WORKING_DIR}/package"
-
-	ls | while read line; do 
-		echo -n "$line\t\t"
-		tar -tzf $line >/dev/null || (echo "FAIL"; rm $line) && echo "OK"
-	done
-	
-	echo `date +%s` > $interval_file
-}
-
-function must_download() {
-	src=$1
-	dst=$2
-	while true; do
-		echo "downloading: $name"
-		wget "$src" -O "$dst" &>/dev/null || true
-		tar -tzf package/$name >/dev/null || rm package/$name && break 
-	done
-}
-
-function hackage_mirror() {
-	local_pklist="/tmp/hackage_local_pklist_$$.list"
-	remote_pklist="/tmp/hackage_remote_pklist_$$.list"
-	
-	cd ${TUNASYNC_WORKING_DIR}
-	mkdir -p package
-
-	echo "Downloading index..."
-	rm index.tar.gz || true
-	axel http://hdiff.luite.com/packages/archive/index.tar.gz -o index.tar.gz > /dev/null
-	
-	echo "building local package list"
-	ls package | sed "s/\.tar\.gz$//" > $local_pklist
-	echo "preferred-versions" >> $local_pklist  # ignore preferred-versions
-	
-	echo "building remote package list"
-	tar ztf index.tar.gz | (cut -d/ -f 1,2 2>/dev/null) | sed 's|/|-|' > $remote_pklist
-	
-	echo "building download list"
-	# substract local list from remote list
-	comm <(sort $remote_pklist) <(sort $local_pklist) -23 | while read pk; do
-		# limit concurrent level
-		bgcount=`jobs | wc -l`
-		while [[ $bgcount -ge 5 ]]; do
-			sleep 0.5
-			bgcount=`jobs | wc -l`
-		done
-		
-		name="$pk.tar.gz"
-		if [ ! -a package/$name ]; then
-			must_download "http://hackage.haskell.org/package/$pk/$name" "package/$name" &
-		else
-			echo "skip existed: $name"
-		fi
-	done
-	
-	# delete redundanty files
-	comm <(sort $remote_pklist) <(sort $local_pklist) -13 | while read pk; do
-		name="$pk.tar.gz"
-		echo "deleting ${name}"
-		rm "package/$name"
-	done
-
-	cp index.tar.gz 00-index.tar.gz
-}
-
-function cleanup () {
-	echo "cleaning up"
-	[[ ! -z $local_pklist ]] && (rm $local_pklist $remote_pklist ; true)
-}
-
-trap cleanup EXIT
-remove_broken 86400
-hackage_mirror 
-
-# vim: ts=4 sts=4 sw=4

scripts/helpers/apt-download

@@ -1,132 +0,0 @@
-#!/bin/bash
-set -e
-LOADED_APT_DOWNLOAD="yes"
-
-function check-and-download() {
-	remote_file=$1
-	local_file=$2
-	wget -q --spider ${remote_file}
-	if [ $? -eq 0 ]; then
-		echo "downloading ${remote_file}"
-		wget -q -N -O ${local_file} ${remote_file}
-		return
-	fi
-	return 0
-}
-
-
-function apt-download-binary() {
-	base_url=$1
-	dist=$2
-	repo=$3
-	arch=$4
-	dest_base_dir=$5
-	if [ -z $dest_base_dir ]; then
-		echo "Destination directory is empty, cannot continue"
-		return 1
-	fi
-	
-	dest_dir="${dest_base_dir}/dists/${dist}"
-	[ ! -d "$dest_dir" ] && mkdir -p "$dest_dir"
-	check-and-download "${base_url}/dists/${dist}/Contents-${arch}.gz" "${dest_dir}/Contents-${arch}.gz" || true
-	check-and-download "${base_url}/dists/${dist}/InRelease"  "${dest_dir}/InRelease" || true
-	check-and-download "${base_url}/dists/${dist}/Release" "${dest_dir}/Release"
-	check-and-download "${base_url}/dists/${dist}/Release.gpg" "${dest_dir}/Release.gpg" || true
-	
-	# Load Package Index URLs from Release file
-	release_file="${dest_dir}/Release"
-	dest_dir="${dest_base_dir}/dists/${dist}/${repo}/binary-${arch}"
-	[ ! -d "$dest_dir" ] && mkdir -p "$dest_dir"
-
-	declare pkgidx_content=""
-	declare cnt_start=false
-	declare -i checksum_len
-	if (grep -e '^SHA256:$' ${release_file} &>/dev/null); then
-		checksum_cmd="sha256sum"; checksum_regex="^SHA256:$"; checksum_len=64
-	elif (grep -e '^SHA1:$' ${release_file} &>/dev/null); then
-		checksum_cmd="sha1sum"; checksum_regex="^SHA1:$"; checksum_len=40
-	elif (grep -e '^MD5Sum:$' ${release_file} &>/dev/null); then
-		checksum_cmd="md5sum"; checksum_regex="^MD5sum:$"; checksum_len=32
-	fi
-
-	while read line; do
-		if [[ ${cnt_start} = true ]]; then
-			read -a tokens <<< $line
-			checksum=${tokens[0]}
-			if [[ ${#checksum} != ${checksum_len} ]]; then
-				break
-			fi
-			filesize=${tokens[1]}
-			filename=${tokens[2]}
-			if [[ "$filename" =~ ${repo}/binary-${arch} ]]; then
-				# Load package list from Packages file
-				pkgidx_file="${dest_base_dir}/dists/${dist}/${filename}"
-				dest_dir=`dirname ${pkgidx_file}`
-				[ ! -d "$dest_dir" ] && mkdir -p "$dest_dir"
-				pkglist_url="${base_url}/dists/${dist}/${filename}"
-				check-and-download "${pkglist_url}" ${pkgidx_file} || true
-				echo "${checksum}  ${pkgidx_file}" | ${checksum_cmd} -c -
-				if [ -z "${pkgidx_content}" -a -f ${pkgidx_file} ]; then
-					echo "getting packages index content"
-					case $filename in
-						"*.bz2")
-							pkgidx_content=`bunzip2 -c ${pkgidx_file}`
-							;;
-						"*.gz")
-							pkgidx_content=`gunzip -c ${pkgidx_file}`
-							;;
-						*)
-							pkgidx_content=`cat ${pkgidx_file}`
-							;;
-					esac
-				fi
-			fi
-		else
-			if [[ "$line" =~ ${checksum_regex} ]]; then
-				cnt_start=true
-			fi
-		fi
-	done < ${release_file}
-
-	if [ -z "${pkgidx_content}" ]; then
-		echo "index is empty, failed"
-		return 1
-	fi
-	
-	# Set checksum method
-	if (echo -e "${pkgidx_content}" | grep -e '^SHA256' &>/dev/null); then
-		checksum_cmd="sha256sum"; checksum_regex="^SHA256"
-	elif (echo -e "${pkgidx_content}" | grep -e '^SHA1' &>/dev/null); then
-		checksum_cmd="sha1sum"; checksum_regex="^SHA1" 
-	elif (echo -e "${pkgidx_content}" | grep -e '^MD5sum' &>/dev/null); then
-		checksum_cmd="md5sum"; checksum_regex="^MD5sum"
-	fi
-
-	# Download packages
-	(echo -e "${pkgidx_content}" | grep -e '^Filename' -e '^Size' -e ${checksum_regex} | cut -d' ' -f 2) | \
-	while read pkg_filename; read pkg_size; read pkg_checksum; do
-		dest_filename="${dest_base_dir}/${pkg_filename}"
-		dest_dir=`dirname ${dest_filename}`
-		[ ! -d "$dest_dir" ] && mkdir -p "$dest_dir"
-		pkg_url="${base_url}/${pkg_filename}"
-		declare downloaded=false
-		if [ -f ${dest_filename} ]; then
-			rsize=`stat -c "%s" ${dest_filename}`
-			if [ ${rsize} -eq ${pkg_size} ]; then
-				downloaded=true
-				echo "Skipping ${pkg_filename}, size ${pkg_size}"
-			fi
-		fi
-		while [ $downloaded != true ]; do
-			echo "downloading ${pkg_url}"
-			wget -q -O ${dest_filename} ${pkg_url} && {
-				echo "${pkg_checksum}  ${dest_filename}" | ${checksum_cmd} -c - && downloaded=true # two space for md5sum/sha1sum/sha256sum check format
-			}
-		done
-	done
-
-	echo "Mirroring ${base_url} ${dist}, ${repo}, ${arch} done!"
-
-}
-
-# vim: ts=4 sts=4 sw=4

scripts/homebrew.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-if [ ! -d "$TUNASYNC_WORKING_DIR" ]; then
-	echo "Directory not exists, fail"
-	exit 1	
-fi
-
-function update_homebrew_git() {
-	repo_dir="$1"
-	cd $repo_dir
-	echo "==== SYNC $repo_dir START ===="
-	/usr/bin/timeout -s INT 3600 git remote -v update
-	echo "==== SYNC $repo_dir DONE ===="
-}
-
-update_homebrew_git "$TUNASYNC_WORKING_DIR/homebrew.git"
-update_homebrew_git "$TUNASYNC_WORKING_DIR/homebrew-python.git"
-update_homebrew_git "$TUNASYNC_WORKING_DIR/homebrew-science.git"

scripts/linux.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-if [ ! -d "$TUNASYNC_WORKING_DIR" ]; then
-	echo "Directory not exists, fail"
-	exit 1	
-fi
-
-function update_linux_git() {
-	cd $TUNASYNC_WORKING_DIR
-	/usr/bin/timeout -s INT 3600 git remote -v update
-}
-
-update_linux_git

scripts/lxc-images.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-function sync_lxc_images() {
-	repo_url="$1"
-	repo_dir="$2"
-
-	[ ! -d "$repo_dir" ] && mkdir -p "$repo_dir"
-	cd $repo_dir
-
-	# lftp "${repo_url}/" -e "mirror --verbose --log=${TUNASYNC_LOG_FILE} --exclude-glob='*/SRPMS/*' -P 5 --delete --only-newer; bye"
-	lftp "${repo_url}/" -e "mirror --verbose -P 5 --delete --only-newer; bye"
-}
-
-
-sync_lxc_images "http://images.linuxcontainers.org/images" "${TUNASYNC_WORKING_DIR}/images"
-sync_lxc_images "http://images.linuxcontainers.org/meta" "${TUNASYNC_WORKING_DIR}/meta"

scripts/mongodb.sh

@@ -1,88 +0,0 @@
-#!/bin/bash
-set -e
-
-_here=`dirname $(realpath $0)`
-. ${_here}/helpers/apt-download
-
-[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1)
-
-BASE_PATH="${TUNASYNC_WORKING_DIR}"
-
-YUM_PATH="${BASE_PATH}/yum"
-APT_PATH="${BASE_PATH}/apt"
-
-UBUNTU_VERSIONS=("trusty" "precise")
-DEBIAN_VERSIONS=("wheezy")
-MONGO_VERSIONS=("3.2" "3.0")
-STABLE_VERSION="3.2"
-
-UBUNTU_PATH="${APT_PATH}/ubuntu"
-DEBIAN_PATH="${APT_PATH}/debian"
-
-mkdir -p $UBUNTU_PATH $DEBIAN_PATH $YUM_PATH
-
-cache_dir="/tmp/yum-mongodb-cache/"
-cfg="/tmp/mongodb-yum.conf"
-cat <<EOF > ${cfg}
-[main]
-keepcache=0
-
-EOF
-
-for mgver in ${MONGO_VERSIONS[@]}; do
-cat <<EOF >> ${cfg}
-[el6-${mgver}]
-name=el6-${mgver}
-baseurl=https://repo.mongodb.org/yum/redhat/6/mongodb-org/${mgver}/x86_64/
-repo_gpgcheck=0
-gpgcheck=0
-enabled=1
-sslverify=0
-
-[el7-${mgver}]
-name=el7-${mgver}
-baseurl=https://repo.mongodb.org/yum/redhat/7/mongodb-org/${mgver}/x86_64/
-repo_gpgcheck=0
-gpgcheck=0
-enabled=1
-sslverify=0
-EOF
-done
-
-reposync -c $cfg -d -p ${YUM_PATH} -e $cache_dir
-for mgver in ${MONGO_VERSIONS[@]}; do
-	createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el6-$mgver/ ${YUM_PATH}/el6-$mgver/
-	createrepo --update -v -c $cache_dir -o ${YUM_PATH}/el7-$mgver/ ${YUM_PATH}/el7-$mgver/
-done
-
-[ -e ${YUM_PATH}/el6 ] || (cd ${YUM_PATH}; ln -s el6-${STABLE_VERSION} el6)
-[ -e ${YUM_PATH}/el7 ] || (cd ${YUM_PATH}; ln -s el7-${STABLE_VERSION} el7)
-
-rm $cfg
-
-base_url="http://repo.mongodb.org/apt/ubuntu"
-for ubver in ${UBUNTU_VERSIONS[@]}; do
-	for mgver in ${MONGO_VERSIONS[@]}; do
-		version="$ubver/mongodb-org/$mgver"
-		apt-download-binary ${base_url} "$version" "multiverse" "amd64" "${UBUNTU_PATH}" || true
-		apt-download-binary ${base_url} "$version" "multiverse" "i386" "${UBUNTU_PATH}" || true
-	done
-	mg_basepath="${UBUNTU_PATH}/dists/$ubver/mongodb-org"
-	[ -e ${mg_basepath}/stable ] || (cd ${mg_basepath}; ln -s ${STABLE_VERSION} stable)
-done
-echo "Ubuntu finished"
-
-base_url="http://repo.mongodb.org/apt/debian"
-for dbver in ${DEBIAN_VERSIONS[@]}; do
-	for mgver in ${MONGO_VERSIONS[@]}; do
-		version="$dbver/mongodb-org/$mgver"
-		apt-download-binary ${base_url} "$version" "main" "amd64" "${DEBIAN_PATH}" || true
-		apt-download-binary ${base_url} "$version" "main" "i386" "${DEBIAN_PATH}" || true
-	done
-	mg_basepath="${DEBIAN_PATH}/dists/$dbver/mongodb-org"
-	[ -e ${mg_basepath}/stable ] || (cd ${mg_basepath}; ln -s ${STABLE_VERSION} stable)
-done
-echo "Debian finished"
-
-
-# vim: ts=4 sts=4 sw=4

scripts/nodesource.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-function sync_nodesource() {
-	repo_url="$1"
-	repo_dir="$2"
-
-	[ ! -d "$repo_dir" ] && mkdir -p "$repo_dir"
-	cd $repo_dir
-	# lftp "${repo_url}/" -e "mirror --verbose --exclude-glob='*/SRPMS/*' -P 5 --delete --only-newer; bye"
-	lftp "${repo_url}/" -e "mirror --verbose  -P 5 --delete --only-newer; bye"
-}
-
-sync_nodesource "https://deb.nodesource.com/node" "${TUNASYNC_WORKING_DIR}/deb"
-sync_nodesource "https://deb.nodesource.com/node_0.12" "${TUNASYNC_WORKING_DIR}/deb_0.12"
-sync_nodesource "https://deb.nodesource.com/node_4.x" "${TUNASYNC_WORKING_DIR}/deb_4.x"
-sync_nodesource "https://rpm.nodesource.com/pub" "${TUNASYNC_WORKING_DIR}/rpm"
-sync_nodesource "https://rpm.nodesource.com/pub_0.12" "${TUNASYNC_WORKING_DIR}/rpm_0.12"
-sync_nodesource "https://rpm.nodesource.com/pub_4.x" "${TUNASYNC_WORKING_DIR}/rpm_4.x"

scripts/openwrt.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-function sync_openwrt() {
-	repo_url="$1"
-	repo_dir="$2"
-
-	[ ! -d "$repo_dir" ] && mkdir -p "$repo_dir"
-	cd $repo_dir
-	lftp "${repo_url}/" -e "mirror --verbose -P 5 --delete --only-newer; bye"
-}
-
-sync_openwrt "http://downloads.openwrt.org/chaos_calmer/15.05/" "${TUNASYNC_WORKING_DIR}/chaos_calmer/15.05"
-sync_openwrt "http://downloads.openwrt.org/snapshots/trunk/" "${TUNASYNC_WORKING_DIR}/snapshots/trunk"

scripts/pypi.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-if [ ! -d "$TUNASYNC_WORKING_DIR" ]; then
-	echo "Directory not exists, fail"
-	exit 1	
-fi
-
-echo "Syncing to $TUNASYNC_WORKING_DIR"
-
-/usr/bin/timeout -s INT 3600 /home/tuna/.virtualenvs/bandersnatch/bin/bandersnatch -c /etc/bandersnatch.conf mirror || exit 1

scripts/repo-ck.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-function sync_repo_ck() {
-	repo_url="$1"
-	repo_dir="$2"
-
-	[ ! -d "$repo_dir" ] && mkdir -p "$repo_dir"
-	cd $repo_dir
-	lftp "${repo_url}/" -e 'mirror -v -P 5 --delete --only-missing --only-newer --no-recursion; bye'
-	wget "${repo_url}/repo-ck.db" -O "repo-ck.db"
-	wget "${repo_url}/repo-ck.files" -O "repo-ck.files"
-}
-
-UPSTREAM="http://repo-ck.com"
-
-sync_repo_ck "${UPSTREAM}/x86_64" "${TUNASYNC_WORKING_DIR}/x86_64"
-sync_repo_ck "${UPSTREAM}/i686" "${TUNASYNC_WORKING_DIR}/i686"

scripts/termux.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-set -e
-
-_here=`dirname $(realpath $0)`
-. ${_here}/helpers/apt-download
-[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1)
-
-BASE_PATH="${TUNASYNC_WORKING_DIR}"
-
-base_url="http://apt.termux.com"
-ARCHES=("aarch64" "all" "arm" "i686")
-for arch in ${ARCHES[@]}; do
-	echo "start syncing: ${arch}"
-	apt-download-binary "${base_url}" "stable" "main" "${arch}" "${BASE_PATH}" || true
-done
-echo "finished"

scripts/ubuntu_debmirror.sh

@@ -1,27 +0,0 @@
-#!/bin/bash
-SYNC_FILES="$TUNASYNC_WORKING_DIR"
-# SYNC_FILES="/srv/mirror_disk/ubuntu/_working/"
-#LOG_FILE="$TUNASYNC_LOG_FILE"
-
-# [ -f $SYNC_LOCK ] && exit 1
-# touch $SYNC_LOCK
-
-
-echo ">> Starting sync on $(date --rfc-3339=seconds)"
-
-arch="i386,amd64"
-sections="main,main/debian-installer,multiverse,multiverse/debian-installer,restricted,restricted/debian-installer,universe,universe/debian-installer"
-dists="precise,precise-backports,precise-proposed,precise-updates,precise-security,trusty,trusty-backports,trusty-proposed,trusty-updates,trusty-security"
-server="$1"
-inPath="/ubuntu"
-proto="rsync"
-outpath="$SYNC_FILES"
-rsyncOpt='-6 -aIL --partial'
-
-debmirror -h $server --no-check-gpg -a $arch -s $sections -d $dists -r $inPath -e $proto --rsync-options "$rsyncOpt" --verbose $outpath
-
-date --rfc-3339=seconds > "$SYNC_FILES/lastsync"
-echo ">> Finished sync on $(date --rfc-3339=seconds)"
-
-# rm -f "$SYNC_LOCK"
-exit 0

worker/cgroup.go

@@ -116,6 +116,7 @@ func (c *cgroupHook) killAll() error {
 			return nil
 		}
 		for _, pid := range taskList {
+			// TODO: deal with defunct processes
 			logger.Debugf("Killing process: %d", pid)
 			unix.Kill(pid, syscall.SIGKILL)
 		}