本系列文章主要记录公司内网软件源的发展史,这是第2篇镜像2.0时代,第一篇可以参考 内网软件源镜像发展史: Apt-Cacher NG时代
1.0版本 不足之处
- 无法缓存alpine包(可能姿势不对)
- 配置维护比较繁琐
- 存在缓存失败或者缓存文件损坏等
- 内部推广不方便(使用代理)
基于上述第3/4点原因,于是探索其他途径
1.5版本 基于tunasync搭建镜像方案
调研了多数开源镜像站点的技术之后,最终决定使用清华大学开源软件镜像站开发的镜像管理器 tunasync 来搭建内网软件镜像服务,毕竟使用成熟的镜像管理器还是很省时省力的。
准备工作
2核8G
50G系统盘2.0T存储盘
同步
基于考虑先测试同步alpine镜像,但是国内目前支持 rsync 服务的镜像源不多,这里以清华源为测试目录
# 查看同步目录
rsync rsync://mirrors.tuna.tsinghua.edu.cn/alpine/
drwxr-xr-x 528 2019/06/20 18:37:55 .
-rw-r--r-- 1,713 2019/10/03 06:00:01 MIRRORS.txt
-rw-r--r-- 11 2019/10/03 18:00:00 last-updated
lrwxrwxrwx 5 2019/06/20 18:37:55 latest-stable
drwxr-xr-x 152 2015/09/30 15:58:27 edge
drwxr-xr-x 120 2012/12/19 23:22:11 v2.4
drwxr-xr-x 96 2012/10/31 20:46:52 v2.5
drwxr-xr-x 96 2013/10/09 21:50:52 v2.6
drwxr-xr-x 128 2014/03/12 20:55:32 v2.7
drwxr-xr-x 120 2014/05/08 06:52:55 v3.0
drwxr-xr-x 144 2015/01/01 15:25:38 v3.1
drwxr-xr-x 128 2019/06/01 02:14:49 v3.10
drwxr-xr-x 96 2015/04/24 17:24:18 v3.2
drwxr-xr-x 184 2015/12/21 22:43:39 v3.3
drwxr-xr-x 128 2016/04/21 20:39:47 v3.4
drwxr-xr-x 128 2016/11/17 00:01:21 v3.5
drwxr-xr-x 128 2017/04/20 18:47:03 v3.6
drwxr-xr-x 128 2017/11/24 05:25:39 v3.7
drwxr-xr-x 128 2018/04/27 14:06:29 v3.8
drwxr-xr-x 128 2018/11/16 00:03:51 v3.9
同步工具使用tunasync
,原理是使用 rsync 做增量同步.
接下来就是同步操作了,根据文档依次来就ok了。
参考文档github/中文文档
这里我贴下我的测试方式。
[root@repo ~]# cat /etc/rc.local
# info : start mirrors service
# Author : ysicing
# CTime : 2019.09.27
/bin/bash /root/bin/tunasync_mirrors.sh s
/bin/bash /root/bin/tunasync_mirrors.sh c
[root@repo ~]# cat /root/bin/tunasync_mirrors.sh
#!/bin/bash
# 同步清华源
type=${1:-s}
if [ "$type" == "s" ]; then
/usr/local/bin/tunasync manager -c /home/mirrors/tunasync/conf/manager.conf >> /data/logs/tunasync/manager.log &
else
/usr/local/bin/tunasync worker -c /home/mirrors/tunasync/conf/worker.conf >> /data/logs/tunasync/worker.log &
fi
# 这里主要贴出alpine配置
[[mirrors]]
name = "alpine"
provider = "rsync"
upstream = "rsync://mirrors.tuna.tsinghua.edu.cn/alpine/"
use_ipv6 = false
exclude_file = "/home/mirrors/tunasync/exclude/alpine.txt"
memory_limit = "256M"
# 只同步3.10-3.8版本的alpine
[root@repo ~]# cat /home/mirrors/tunasync/exclude/alpine.txt
edge/
v2.*/
v3.0/
v3.1/
v3.2/
v3.3/
v3.4/
v3.5/
v3.6/
v3.7/
经过漫长的几个小时,同步完几百G大小alpine,可以完美使用,美滋滋。
但是后面同事说,这没必要全量缓存,可以使用artifactory做cache缓存,是哇,可以哦毕竟上家公司用过,还是很熟悉的,pro版支持包缓存
2.0版本 artifactory缓存
docker-compose.yaml
配置如下,hub.ops.com
内网域名
version: '2.1'
services:
oss:
image: hub.ops.com/soft/artifactory-pro:6.12.2
container_name: oss
volumes:
- /data/oss/data:/var/opt/jfrog/artifactory/data
- /data/oss/logs:/var/opt/jfrog/artifactory/logs
- /data/oss/etc:/var/opt/jfrog/artifactory/etc
network_mode: host
restart: always
nginx:
image: hub.ops.com/common/nginx:1.17.3
container_name: nginx
volumes:
- /var/log/nginx:/var/log/nginx:rw
- ./config:/etc/nginx/conf.d:rw
- ./nginxconfig.io:/etc/nginx/nginxconfig.io:rw
- ./ssl:/etc/nginx/ssl:rw
- ./wwwroot:/var/www:rw
network_mode: host
restart: always
核心nginx配置,域名隐去了。
[root@repo mirrors]# cat config/mirrors.conf
server {
listen 80;
listen [::]:80;
listen 443 ssl http2;
listen [::]:443 ssl http2;
server_name mirrors.com;
index index.html;
root /var/www/mirrors.com/public;
# SSL
ssl_certificate /etc/nginx/ssl/mirrors.com.crt;
ssl_certificate_key /etc/nginx/ssl/mirrors.com.key;
# security
# include security.conf;
location ~ .*\.(html|htm|json|reponew|sh)$ {
root /var/www/mirrors.ops.com/public;
}
location ~ ^/$ {
root /var/www/mirrors.ops.com/public;
}
# reverse proxy
location / {
proxy_pass http://172.16.72.42:8081/artifactory/;
include nginxconfig.io/proxy.conf;
}
# additional config
include nginxconfig.io/general.conf;
}
[root@repo nginxconfig.io]# cat proxy.conf
proxy_http_version 1.1;
proxy_cache_bypass $http_upgrade;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Port $server_port;
[root@repo nginxconfig.io]# cat general.conf
# favicon.ico
location = /favicon.ico {
log_not_found off;
access_log off;
}
# robots.txt
location = /robots.txt {
log_not_found off;
access_log off;
}
# assets, media
location ~* \.(?:css(\.map)?|js(\.map)?|jpe?g|png|gif|ico|cur|heic|webp|tiff?|mp3|m4a|aac|ogg|midi?|wav|mp4|mov|webm|mpe?g|avi|ogv|flv|wmv)$ {
expires 7d;
access_log off;
}
# svg, fonts
location ~* \.(?:svgz?|ttf|ttc|otf|eot|woff2?)$ {
add_header Access-Control-Allow-Origin "*";
expires 7d;
access_log off;
}
# gzip
gzip on;
gzip_vary on;
gzip_proxied any;
gzip_comp_level 6;
gzip_types text/plain text/css text/xml application/json application/javascript application/rss+xml application/atom+xml image/svg+xml;
缓存配置路径为
http://<artifactory节点ip>:8081/artifactory/webapp/#/admin/repositories/remote
# 用户名
admin/password
参考了网易源的代码。
上述是基于已有授权的情况,如果没有授权怎么办,仅供参考不提倡
版权属于:ysicing
本文链接:https://godu.dev/posts/mirrors-package/
转载时须注明出处及本声明