update scripts

scripts/android.sh

@@ -2,7 +2,15 @@
 
 function sync_android() {
 	cd $TUNASYNC_WORKING_DIR
-	/usr/local/bin/android-repo sync
+	/usr/local/bin/android-repo sync -f
+}
+
+function update_server_info() {
+	for repo in $(find $TUNASYNC_WORKING_DIR -type d -not -path "*/.repo/*" -name "*.git")
+	do
+		cd $repo
+		git update-server-info
+	done
 }
 
 sync_android

scripts/gen_debian_exclude.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python
 
-ARCH_EXCLUDE = ['armel', 'arm64', 'alpha', 'hurd-i386', 'ia64', 'kfreebsd-amd64', 'kfreebsd-i386', 'mips', 'powerpc', 'ppc64el',  's390', 's390x', 'sparc']
+ARCH_EXCLUDE = ['armel', 'alpha', 'hurd-i386', 'ia64', 'kfreebsd-amd64', 'kfreebsd-i386', 'mips', 'powerpc', 'ppc64el',  's390', 's390x', 'sparc']
 
 CONTENT_EXCLUDE = ['binary-{arch}', 'installer-{arch}', 'Contents-{arch}.gz', 'Contents-udeb-{arch}.gz', 'Contents-{arch}.diff', 'arch-{arch}.files', 'arch-{arch}.list.gz', '*_{arch}.deb', '*_{arch}.udeb', '*_{arch}.changes']
 
 with open("debian-exclude.txt", 'wb') as f:
+	f.write(".~tmp~/\n")
+	f.write(".*\n")
 	for arch in ARCH_EXCLUDE:
 		for content in CONTENT_EXCLUDE:
 			f.write(content.format(arch=arch))

scripts/gen_kali_exclude.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+ARCH_EXCLUDE = ['armel', 'armhf']
+
+CONTENT_EXCLUDE = ['binary-{arch}', 'installer-{arch}', 'Contents-{arch}.gz', 'Contents-udeb-{arch}.gz', 'Contents-{arch}.diff', 'arch-{arch}.files', 'arch-{arch}.list.gz', '*_{arch}.deb', '*_{arch}.udeb', '*_{arch}.changes']
+
+with open("kali-exclude.txt", 'wb') as f:
+	f.write(".~tmp~/\n")
+	f.write(".*\n")
+	for arch in ARCH_EXCLUDE:
+		for content in CONTENT_EXCLUDE:
+			f.write(content.format(arch=arch))
+			f.write('\n')

scripts/gen_ubuntu_ports_exclude.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+ARCH_EXCLUDE = ['powerpc', 'ppc64el', 'ia64', 'sparc', 'armel']
+
+CONTENT_EXCLUDE = ['binary-{arch}', 'installer-{arch}', 'Contents-{arch}.gz', 'Contents-udeb-{arch}.gz', 'Contents-{arch}.diff', 'arch-{arch}.files', 'arch-{arch}.list.gz', '*_{arch}.deb', '*_{arch}.udeb', '*_{arch}.changes']
+
+with open("ubuntu-ports-exclude.txt", 'wb') as f:
+	f.write(".~tmp~/\n")
+	f.write(".*\n")
+	for arch in ARCH_EXCLUDE:
+		for content in CONTENT_EXCLUDE:
+			f.write(content.format(arch=arch))
+			f.write('\n')

scripts/homebrew.sh

@@ -5,8 +5,13 @@ if [ ! -d "$TUNASYNC_WORKING_DIR" ]; then
 fi
 
 function update_homebrew_git() {
-	cd $TUNASYNC_WORKING_DIR
+	repo_dir="$1"
+	cd $repo_dir
+	echo "==== SYNC $repo_dir START ===="
 	/usr/bin/timeout -s INT 3600 git remote -v update
+	echo "==== SYNC $repo_dir DONE ===="
 }
 
-update_homebrew_git
+update_homebrew_git "$TUNASYNC_WORKING_DIR/homebrew.git"
+update_homebrew_git "$TUNASYNC_WORKING_DIR/homebrew-python.git"
+update_homebrew_git "$TUNASYNC_WORKING_DIR/homebrew-science.git"

scripts/lxc-images.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+function sync_lxc_images() {
+	repo_url="$1"
+	repo_dir="$2"
+	cd $repo_dir
+
+	# lftp "${repo_url}/" -e "mirror --verbose --log=${TUNASYNC_LOG_FILE} --exclude-glob='*/SRPMS/*' -P 5 --delete --only-newer; bye"
+	lftp "${repo_url}/" -e "mirror --verbose --exclude lxd/ -P 5 --delete --only-newer; bye"
+}
+
+
+sync_lxc_images "http://images.linuxcontainers.org/" "${TUNASYNC_WORKING_DIR}/"

scripts/nodesource.sh

@@ -1,11 +1,13 @@
 #!/bin/bash
+
 function sync_repo_ck() {
-        repo_url="$1"
-        repo_dir="$2"
+	repo_url="$1"
+	repo_dir="$2"
 
-        [ ! -d "$repo_dir" ] && mkdir -p "$repo_dir"
-        cd $repo_dir
-        lftp "${repo_url}/" -e "mirror --verbose  -P 5 --delete --only-newer; bye"
+	[ ! -d "$repo_dir" ] && mkdir -p "$repo_dir"
+	cd $repo_dir
+	# lftp "${repo_url}/" -e "mirror --verbose --log=${TUNASYNC_LOG_FILE} --exclude-glob='*/SRPMS/*' -P 5 --delete --only-newer; bye"
+	lftp "${repo_url}/" -e "mirror --verbose  -P 5 --delete --only-newer; bye"
 }
 
 sync_repo_ck "https://deb.nodesource.com/node" "${TUNASYNC_WORKING_DIR}/deb"

scripts/pypi.sh

@@ -4,4 +4,6 @@ if [ ! -d "$TUNASYNC_WORKING_DIR" ]; then
 	exit 1	
 fi
 
-/usr/bin/timeout -s INT 3600 /home/tuna/.virtualenvs/bandersnatch/bin/bandersnatch -c /etc/bandersnatch.conf mirror 
+echo "Syncing to $TUNASYNC_WORKING_DIR"
+
+/usr/bin/timeout -s INT 3600 /home/tuna/.virtualenvs/bandersnatch/bin/bandersnatch -c /etc/bandersnatch.conf mirror || exit 1

scripts/repo-ck.sh

@@ -6,12 +6,12 @@ function sync_repo_ck() {
 
 	[ ! -d "$repo_dir" ] && mkdir -p "$repo_dir"
 	cd $repo_dir
-	lftp "${repo_url}/" -e 'mirror -v -P 5 --delete --only-missing --only-newer --no-recursion; bye' || return 1
-	wget "${repo_url}/repo-ck.db" -O "repo-ck.db" || return 1
+	lftp "${repo_url}/" -e 'mirror -v -P 5 --delete --only-missing --only-newer --no-recursion; bye'
+	wget "${repo_url}/repo-ck.db" -O "repo-ck.db"
 	wget "${repo_url}/repo-ck.files" -O "repo-ck.files"
 }
 
-stat=0
-sync_repo_ck "${TUNASYNC_UPSTREAM_URL}/x86_64" "${TUNASYNC_WORKING_DIR}/x86_64" || stat=1
-sync_repo_ck "${TUNASYNC_UPSTREAM_URL}/i686" "${TUNASYNC_WORKING_DIR}/i686" || stat=1
-exit $stat
+UPSTREAM="http://repo-ck.com"
+
+sync_repo_ck "${UPSTREAM}/x86_64" "${TUNASYNC_WORKING_DIR}/x86_64"
+sync_repo_ck "${UPSTREAM}/i686" "${TUNASYNC_WORKING_DIR}/i686"

scripts/ubuntu_debmirror.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+SYNC_FILES="$TUNASYNC_WORKING_DIR"
+# SYNC_FILES="/srv/mirror_disk/ubuntu/_working/"
+#LOG_FILE="$TUNASYNC_LOG_FILE"
+
+# [ -f $SYNC_LOCK ] && exit 1
+# touch $SYNC_LOCK
+
+
+echo ">> Starting sync on $(date --rfc-3339=seconds)"
+
+arch="i386,amd64"
+sections="main,main/debian-installer,multiverse,multiverse/debian-installer,restricted,restricted/debian-installer,universe,universe/debian-installer"
+dists="precise,precise-backports,precise-proposed,precise-updates,precise-security,trusty,trusty-backports,trusty-proposed,trusty-updates,trusty-security"
+server="$1"
+inPath="/ubuntu"
+proto="rsync"
+outpath="$SYNC_FILES"
+rsyncOpt='-6 -aIL --partial'
+
+debmirror -h $server --no-check-gpg -a $arch -s $sections -d $dists -r $inPath -e $proto --rsync-options "$rsyncOpt" --verbose $outpath
+
+date --rfc-3339=seconds > "$SYNC_FILES/lastsync"
+echo ">> Finished sync on $(date --rfc-3339=seconds)"
+
+# rm -f "$SYNC_LOCK"
+exit 0