add apt-download helper, updated gitlab-ce mirror script

scripts/gitlab-ce.sh

@@ -1,14 +1,19 @@
 #!/bin/bash
 set -e
 
-BASE_PATH="${TUNASYNC_WORKING_DIR}"
+_here=`dirname $(realpath $0)`
+. ${_here}/helpers/apt-download
+
+[ -z "${LOADED_APT_DOWNLOAD}" ] && (echo "failed to load apt-download"; exit 1)
 
+BASE_PATH="${TUNASYNC_WORKING_DIR}"
 
-APT_PATH="${BASE_PATH}/apt"
 YUM_PATH="${BASE_PATH}/yum"
 
 UBUNTU_VERSIONS=("trusty" "wily")
 DEBIAN_VERSIONS=("wheezy" "jessie" "stretch")
+UBUNTU_PATH="${BASE_PATH}/ubuntu/"
+DEBIAN_PATH="${BASE_PATH}/debian/"
 
 mkdir -p $UBUNTU_PATH $DEBIAN_PATH $YUM_PATH
 
@@ -35,44 +40,23 @@ sslverify=0
 EOF
 
 reposync -c $cfg -d -p ${YUM_PATH}
-createrepo -o ${YUM_PATH}/el6 ${YUM_PATH}/el6
-createrepo -o ${YUM_PATH}/el7 ${YUM_PATH}/el7
+createrepo --update -o ${YUM_PATH}/el6 ${YUM_PATH}/el6
+createrepo --update -o ${YUM_PATH}/el7 ${YUM_PATH}/el7
 rm $cfg
 
-
-cfg="/tmp/gitlab-ce-ubuntu.list"
-cat << EOF > ${cfg}
-set mirror_path ${APT_PATH}
-set nthreds 5
-set _tilde 0
-
-EOF
+base_url="https://packages.gitlab.com/gitlab/gitlab-ce/ubuntu"
 for version in ${UBUNTU_VERSIONS[@]}; do
-		echo "deb https://packages.gitlab.com/gitlab/gitlab-ce/ubuntu/ $version main" >> $cfg
-		echo "deb-i386 https://packages.gitlab.com/gitlab/gitlab-ce/ubuntu/ $version main" >> $cfg
-		echo "deb-amd64 https://packages.gitlab.com/gitlab/gitlab-ce/ubuntu/ $version main" >> $cfg
-		echo "deb-src https://packages.gitlab.com/gitlab/gitlab-ce/ubuntu/ $version main" >> $cfg
+	apt-download-binary ${base_url} "$version" "main" "amd64" "${UBUNTU_PATH}"
+	apt-download-binary ${base_url} "$version" "main" "i386" "${UBUNTU_PATH}"
 done
+echo "Ubuntu finished"
 
-apt-mirror $cfg
-rm $cfg
-
-
-cfg="/tmp/gitlab-ce-debian.list"
-cat << EOF > ${cfg}
-set mirror_path ${APT_PATH}
-set nthreds 5
-set _tilde 0
-
-EOF
+base_url="https://packages.gitlab.com/gitlab/gitlab-ce/debian"
 for version in ${DEBIAN_VERSIONS[@]}; do
-		echo "deb https://packages.gitlab.com/gitlab/gitlab-ce/debian/ $version main" >> $cfg
-		echo "deb-i386 https://packages.gitlab.com/gitlab/gitlab-ce/debian/ $version main" >> $cfg
-		echo "deb-amd64 https://packages.gitlab.com/gitlab/gitlab-ce/debian/ $version main" >> $cfg
-		echo "deb-src https://packages.gitlab.com/gitlab/gitlab-ce/debian/ $version main" >> $cfg
+	apt-download-binary ${base_url} "$version" "main" "amd64" "${DEBIAN_PATH}"
+	apt-download-binary ${base_url} "$version" "main" "i386" "${DEBIAN_PATH}"
 done
+echo "Debian finished"
 
-apt-mirror $cfg
-rm $cfg
 
 # vim: ts=4 sts=4 sw=4

scripts/helpers/apt-download

@@ -0,0 +1,78 @@
+#!/bin/bash
+set -e
+LOADED_APT_DOWNLOAD="yes"
+
+function check-and-download() {
+	remote_file=$1
+	local_file=$2
+	wget -q --spider ${remote_file}
+	if [ $? -eq 0 ]; then
+		echo "downloading ${remote_file}"
+		wget -q -N -O ${local_file} ${remote_file}
+		return
+	fi
+	return 0
+}
+
+
+function apt-download-binary() {
+	base_url=$1
+	dist=$2
+	repo=$3
+	arch=$4
+	dest_base_dir=$5
+	
+	dest_dir="${dest_base_dir}/dists/${dist}"
+	[ ! -d "$dest_dir" ] && mkdir -p "$dest_dir"
+	check-and-download "${base_url}/dists/${dist}/Contents-${arch}.gz" "${dest_dir}/Contents-${arch}.gz"
+	check-and-download "${base_url}/dists/${dist}/InRelease"  "${dest_dir}/InRelease" || true
+	check-and-download "${base_url}/dists/${dist}/Release" "${dest_dir}/Release"
+	check-and-download "${base_url}/dists/${dist}/Release.gpg" "${dest_dir}/Release.gpg" || true
+
+	dest_dir="${dest_base_dir}/dists/${dist}/${repo}/binary-${arch}"
+	[ ! -d "$dest_dir" ] && mkdir -p "$dest_dir"
+
+	pkgidxes=('Packages' 'Packages.gz' 'Packages.bz2' 'Packages.xz')
+	pkgidx_content=""
+	for pkgidx in ${pkgidxes[@]}; do
+		pkgidx_file="${dest_dir}/${pkgidx}"
+		pkglist_url="${base_url}/dists/${dist}/${repo}/binary-${arch}/${pkgidx}"
+		check-and-download "${pkglist_url}" ${pkgidx_file} || true
+		if [ -z "${pkgidx_content}" -a -f ${pkgidx_file} ]; then
+			echo "getting packages index content"
+			case $pkgidx in
+				"*.bz2")
+					pkgidx_content=`bunzip2 -c ${pkgidx_file}`
+					;;
+				"*.gz")
+					pkgidx_content=`gunzip -c ${pkgidx_file}`
+					;;
+				*)
+					pkgidx_content=`cat ${pkgidx_file}`
+					;;
+			esac
+		fi
+	done
+
+	if [ -z "${pkgidx_content}" ]; then
+		echo "index is empty, failed"
+		return 1
+	fi
+	
+	(echo -e "${pkgidx_content}" | grep '^Filename' | cut -d' ' -f 2) | while read pkg_filename; do
+		dest_filename="${dest_base_dir}/${pkg_filename}"
+		dest_dir=`dirname ${dest_filename}`
+		[ ! -d "$dest_dir" ] && mkdir -p "$dest_dir"
+		pkg_url="${base_url}/${pkg_filename}"
+
+		if [ -f ${dest_filename} ]; then
+			echo "Skipping ${pkg_filename}"
+		else
+			echo "downloading ${pkg_url}"
+			# touch ${dest_filename}
+			wget -q -O ${dest_filename} ${pkg_url} || true
+		fi
+
+	done
+
+}