最近一些客户反馈,希望能够在 Windows 系统使用 TCR 镜像仓库。以及一些内部用户场景,需要使用 Windows 镜像,并且希望能够有代理服务,能够缓存加速 Windows Core 镜像。特此记录下操作过程和笔记。

环境准备

部署 Containerd

参考文章:

Windows Containers on Windows 10 without Docker (using Containerd)

配置文件

1
2
3
4
5
6
# crictl.yaml

runtime-endpoint: npipe://./pipe/containerd-containerd
image-endpoint: npipe://./pipe/containerd-containerd
timeout: 10
debug: true

Containerd 配置 Mirror

参考文档: containerd/registry.md at main · containerd/containerd (github.com)

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
# containerd.toml

disabled_plugins = []
imports = []
oom_score = 0
plugin_dir = ""
required_plugins = []
root = "C:\\ProgramData\\containerd\\root"
state = "C:\\ProgramData\\containerd\\state"
temp = ""
version = 2

[cgroup]
  path = ""

[debug]
  address = ""
  format = ""
  gid = 0
  level = ""
  uid = 0

[grpc]
  address = "\\\\.\\pipe\\containerd-containerd"
  gid = 0
  max_recv_message_size = 16777216
  max_send_message_size = 16777216
  tcp_address = ""
  tcp_tls_ca = ""
  tcp_tls_cert = ""
  tcp_tls_key = ""
  uid = 0

[metrics]
  address = ""
  grpc_histogram = false

[plugins]

  [plugins."io.containerd.gc.v1.scheduler"]
    deletion_threshold = 0
    mutation_threshold = 100
    pause_threshold = 0.02
    schedule_delay = "0s"
    startup_delay = "100ms"

  [plugins."io.containerd.grpc.v1.cri"]
    device_ownership_from_security_context = false
    disable_apparmor = false
    disable_cgroup = false
    disable_hugetlb_controller = false
    disable_proc_mount = false
    disable_tcp_service = true
    enable_selinux = false
    enable_tls_streaming = false
    enable_unprivileged_icmp = false
    enable_unprivileged_ports = false
    ignore_image_defined_volumes = false
    max_concurrent_downloads = 3
    max_container_log_line_size = 16384
    netns_mounts_under_state_dir = false
    restrict_oom_score_adj = false
    sandbox_image = "k8s.gcr.io/pause:3.6"
    selinux_category_range = 0
    stats_collect_period = 10
    stream_idle_timeout = "4h0m0s"
    stream_server_address = "127.0.0.1"
    stream_server_port = "0"
    systemd_cgroup = false
    tolerate_missing_hugetlb_controller = false
    unset_seccomp_profile = ""

    [plugins."io.containerd.grpc.v1.cri".cni]
      bin_dir = "C:\\Program Files\\containerd\\cni\\bin"
      conf_dir = "C:\\Program Files\\containerd\\cni\\conf"
      conf_template = ""
      ip_pref = ""
      max_conf_num = 1

    [plugins."io.containerd.grpc.v1.cri".containerd]
      default_runtime_name = "runhcs-wcow-process"
      disable_snapshot_annotations = false
      discard_unpacked_layers = false
      ignore_rdt_not_enabled_errors = false
      no_pivot = false
      snapshotter = "windows"

      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
        base_runtime_spec = ""
        cni_conf_dir = ""
        cni_max_conf_num = 0
        container_annotations = []
        pod_annotations = []
        privileged_without_host_devices = false
        runtime_engine = ""
        runtime_path = ""
        runtime_root = ""
        runtime_type = ""

        [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]

      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]

        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process]
          base_runtime_spec = ""
          cni_conf_dir = ""
          cni_max_conf_num = 0
          container_annotations = []
          pod_annotations = []
          privileged_without_host_devices = false
          runtime_engine = ""
          runtime_path = ""
          runtime_root = ""
          runtime_type = "io.containerd.runhcs.v1"

          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process.options]

      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
        base_runtime_spec = ""
        cni_conf_dir = ""
        cni_max_conf_num = 0
        container_annotations = []
        pod_annotations = []
        privileged_without_host_devices = false
        runtime_engine = ""
        runtime_path = ""
        runtime_root = ""
        runtime_type = ""

        [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]

    [plugins."io.containerd.grpc.v1.cri".image_decryption]
      key_model = "node"

    [plugins."io.containerd.grpc.v1.cri".registry]
      config_path = ""

      [plugins."io.containerd.grpc.v1.cri".registry.auths]

      [plugins."io.containerd.grpc.v1.cri".registry.configs]

      [plugins."io.containerd.grpc.v1.cri".registry.headers]

      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]

    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
      tls_cert_file = ""
      tls_key_file = ""

  [plugins."io.containerd.internal.v1.opt"]
    path = "C:\\ProgramData\\containerd\\root\\opt"

  [plugins."io.containerd.internal.v1.restart"]
    interval = "10s"

  [plugins."io.containerd.internal.v1.tracing"]
    sampling_ratio = 1.0
    service_name = "containerd"

  [plugins."io.containerd.metadata.v1.bolt"]
    content_sharing_policy = "shared"

  [plugins."io.containerd.runtime.v2.task"]
    platforms = ["windows/amd64", "linux/amd64"]
    sched_core = false

  [plugins."io.containerd.service.v1.diff-service"]
    default = ["windows", "windows-lcow"]

  [plugins."io.containerd.service.v1.tasks-service"]
    rdt_config_file = ""

  [plugins."io.containerd.tracing.processor.v1.otlp"]
    endpoint = ""
    insecure = false
    protocol = ""

[proxy_plugins]

[stream_processors]

  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
    accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
    args = ["--decryption-keys-path", "C:\\Program Files\\containerd\\ocicrypt\\keys"]
    env = ["OCICRYPT_KEYPROVIDER_CONFIG=C:\\Program Files\\containerd\\ocicrypt\\ocicrypt_keyprovider.conf"]
    path = "ctd-decoder"
    returns = "application/vnd.oci.image.layer.v1.tar"

  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
    accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
    args = ["--decryption-keys-path", "C:\\Program Files\\containerd\\ocicrypt\\keys"]
    env = ["OCICRYPT_KEYPROVIDER_CONFIG=C:\\Program Files\\containerd\\ocicrypt\\ocicrypt_keyprovider.conf"]
    path = "ctd-decoder"
    returns = "application/vnd.oci.image.layer.v1.tar+gzip"

[timeouts]
  "io.containerd.timeout.bolt.open" = "0s"
  "io.containerd.timeout.shim.cleanup" = "5s"
  "io.containerd.timeout.shim.load" = "5s"
  "io.containerd.timeout.shim.shutdown" = "3s"
  "io.containerd.timeout.task.state" = "2s"

[ttrpc]
  address = ""
  gid = 0
  uid = 0

[plugins."io.containerd.grpc.v1.cri".registry.mirrors."mcr.microsoft.com"]
endpoint = ["http://49.232.217.60:5000"]

#[plugins.cri.registry.mirrors]
#[plugins.cri.registry.mirrors."mcr.microsoft.com"]
#endpoint = ["http://49.232.217.60:5000"]

部署 Registry Mirror

配置文件

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
version: 0.1
log:
  level: debug
  fields:
    service: registry

proxy:
  remoteurl: https://mcr.microsoft.com

storage:
  cache:
    blobdescriptor: inmemory
  filesystem:
    rootdirectory: /var/lib/registry
http:
  addr: :5000
  headers:
    X-Content-Type-Options: [nosniff]
health:
  storagedriver:
    enabled: true
    interval: 10s
    threshold: 3

启动 Mirror 容器

1
docker run --name lab -p 5000:5000 -v data:/var/lib/registry -v config.yml:/etc/docker/registry/config.yml ccr.ccs.tencentyun.com/library/registry

测试记录

服务端

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
ime="2022-02-18T06:33:35.858558666Z" level=warning msg="No HTTP secret provided - generated random secret. This may cause problems with uploads if multiple registries are behind a load-balancer. To provide a shared secret, fill in http.secret in the configuration file or set the REGISTRY_HTTP_SECRET environment variable." go.version=go1.11.2 instance.id=1455028d-da6d-4743-9ad5-940bba016f10 service=registry version=v2.7.1
time="2022-02-18T06:33:35.85863219Z" level=info msg="redis not configured" go.version=go1.11.2 instance.id=1455028d-da6d-4743-9ad5-940bba016f10 service=registry version=v2.7.1
time="2022-02-18T06:33:35.860982269Z" level=info msg="Starting upload purge in 53m0s" go.version=go1.11.2 instance.id=1455028d-da6d-4743-9ad5-940bba016f10 service=registry version=v2.7.1
time="2022-02-18T06:33:35.872414901Z" level=info msg="using inmemory blob descriptor cache" go.version=go1.11.2 instance.id=1455028d-da6d-4743-9ad5-940bba016f10 service=registry version=v2.7.1
time="2022-02-18T06:33:35.87439159Z" level=info msg="listening on [::]:5000" go.version=go1.11.2 instance.id=1455028d-da6d-4743-9ad5-940bba016f10 service=registry version=v2.7.1
time="2022-02-18T06:34:00.48305885Z" level=error msg="response completed with error" err.code="manifest unknown" err.detail="unknown tag=ltsc2019" err.message="manifest unknown" go.version=go1.11.2 http.request.host="49.232.217.60:5000" http.request.id=1b62b54d-553b-4d2e-9487-c0d7fe8f7115 http.request.method=HEAD http.request.remoteaddr="10.88.0.1:60703" http.request.uri="/v2/windows/servercore/manifests/ltsc2019?ns=mcr.microsoft.com" http.request.useragent="containerd/v1.6.0" http.response.contenttype="application/json; charset=utf-8" http.response.duration=1.273899ms http.response.status=404 http.response.written=98 vars.name="windows/servercore" vars.reference=ltsc2019
10.88.0.1 - - [18/Feb/2022:06:34:00 +0000] "HEAD /v2/windows/servercore/manifests/ltsc2019?ns=mcr.microsoft.com HTTP/1.1" 404 98 "" "containerd/v1.6.0"
time="2022-02-18T06:34:01.050691255Z" level=error msg="response completed with error" err.code="manifest unknown" err.detail="unknown manifest name=windows/servercore revision=sha256:d8c2b69511a49e9eb6fd2a17f2727ae011b0382cd11562fa5cb4c70bb460cf12" err.message="manifest unknown" go.version=go1.11.2 http.request.host="49.232.217.60:5000" http.request.id=88e632bc-5eec-49b9-b625-624eaf8b96fa http.request.method=GET http.request.remoteaddr="10.88.0.1:60705" http.request.uri="/v2/windows/servercore/manifests/sha256:d8c2b69511a49e9eb6fd2a17f2727ae011b0382cd11562fa5cb4c70bb460cf12?ns=mcr.microsoft.com" http.request.useragent="containerd/v1.6.0" http.response.contenttype="application/json; charset=utf-8" http.response.duration=1.457987ms http.response.status=404 http.response.written=194 vars.name="windows/servercore" vars.reference="sha256:d8c2b69511a49e9eb6fd2a17f2727ae011b0382cd11562fa5cb4c70bb460cf12"
10.88.0.1 - - [18/Feb/2022:06:34:01 +0000] "GET /v2/windows/servercore/manifests/sha256:d8c2b69511a49e9eb6fd2a17f2727ae011b0382cd11562fa5cb4c70bb460cf12?ns=mcr.microsoft.com HTTP/1.1" 404 194 "" "containerd/v1.6.0"
time="2022-02-18T06:34:04.339934302Z" level=error msg="response completed with error" err.code="manifest unknown" err.detail="unknown manifest name=windows/servercore revision=sha256:596c8a02a8e73b53688052907d121879357d2a9208700306bafaa1ffb533cbab" err.message="manifest unknown" go.version=go1.11.2 http.request.host="49.232.217.60:5000" http.request.id=65d32ca4-113f-4acc-b035-e73baff05c48 http.request.method=GET http.request.remoteaddr="10.88.0.1:60707" http.request.uri="/v2/windows/servercore/manifests/sha256:596c8a02a8e73b53688052907d121879357d2a9208700306bafaa1ffb533cbab?ns=mcr.microsoft.com" http.request.useragent="containerd/v1.6.0" http.response.contenttype="application/json; charset=utf-8" http.response.duration=1.546302ms http.response.status=404 http.response.written=194 vars.name="windows/servercore" vars.reference="sha256:596c8a02a8e73b53688052907d121879357d2a9208700306bafaa1ffb533cbab"
10.88.0.1 - - [18/Feb/2022:06:34:04 +0000] "GET /v2/windows/servercore/manifests/sha256:596c8a02a8e73b53688052907d121879357d2a9208700306bafaa1ffb533cbab?ns=mcr.microsoft.com HTTP/1.1" 404 194 "" "containerd/v1.6.0"
time="2022-02-18T06:34:04.459994991Z" level=error msg="response completed with error" err.code="blob unknown" err.detail=sha256:c703d4d605332bcb954f514c7d2c6cd53ee51024d7a7a3f13eb907abbeedbca2 err.message="blob unknown to registry" go.version=go1.11.2 http.request.host="49.232.217.60:5000" http.request.id=80f71f7f-4fed-4a87-af23-31d88614cde0 http.request.method=GET http.request.remoteaddr="10.88.0.1:60708" http.request.uri="/v2/windows/servercore/blobs/sha256:c703d4d605332bcb954f514c7d2c6cd53ee51024d7a7a3f13eb907abbeedbca2?ns=mcr.microsoft.com" http.request.useragent="containerd/v1.6.0" http.response.contenttype="application/json; charset=utf-8" http.response.duration=1.659114ms http.response.status=404 http.response.written=157 vars.digest="sha256:c703d4d605332bcb954f514c7d2c6cd53ee51024d7a7a3f13eb907abbeedbca2" vars.name="windows/servercore"
10.88.0.1 - - [18/Feb/2022:06:34:04 +0000] "GET /v2/windows/servercore/blobs/sha256:c703d4d605332bcb954f514c7d2c6cd53ee51024d7a7a3f13eb907abbeedbca2?ns=mcr.microsoft.com HTTP/1.1" 404 157 "" "containerd/v1.6.0"

客户端

window-docker-image

结论

2022-02-18

初步看,没法通过一般法的方法设置 MCR 的 Mirror。