Skip to content

[WIP] Booting/testing something in CI #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 14 commits into
base: master
Choose a base branch
from
Open

[WIP] Booting/testing something in CI #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
8a30148
CI: Build .iso (but not -efi.iso) images.
Dec 5, 2017
40ce9c4
CI: Try booting something
Dec 8, 2017
769fa97
Drop some commented info
Jan 12, 2018
4eec22b
Try using machine job type to boot.
Jan 12, 2018
e9d816f
less verbose, reduce noise vs master
Jan 12, 2018
4975d44
use sudo, machine executor does not run as root
Jan 12, 2018
ba058d7
Just test cri-container + bridge while debugging CI
Jan 12, 2018
e39aebe
Skip some more packages while debugging
Jan 12, 2018
769d223
Move workspace to ~ for machine executor
Jan 12, 2018
9b2ee01
fix for shell ~ expansion and set -x
Jan 12, 2018
7a6924c
Use less memory
Jan 12, 2018
1e91ab4
more debug, less vcpus
Jan 12, 2018
457e7d6
Workspace binaries in Linux subdir
Jan 12, 2018
e342651
Try booting with Hyperkit on MacOS
Jan 12, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
287 changes: 208 additions & 79 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ linuxkit_pkg_build: &linuxkit_pkg_build
steps:
- run:
name: Configure $PATH
command: echo 'export PATH=/workspace/bin:$PATH' >> $BASH_ENV
command: echo 'export PATH=/workspace/bin-Linux:$PATH' >> $BASH_ENV
- run:
name: Install packages
# ca-certificates are needed for attach_workspace (and git over https)
Expand Down Expand Up @@ -38,7 +38,7 @@ image_build: &image_build
steps:
- run:
name: Configure $PATH
command: echo 'export PATH=/workspace/bin:$PATH' >> $BASH_ENV
command: echo 'export PATH=/workspace/bin-Linux:$PATH' >> $BASH_ENV
- run:
name: Install packages
# ca-certificates are needed for attach_workspace (and git over https)
Expand Down Expand Up @@ -83,9 +83,73 @@ image_build: &image_build
name: Build images
command: |
mkdir -p /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK
# KUBE_FORMATS="iso-efi iso-bios" are much slower (especially for RUNTIME=docker) to build than tar.
# So for now just build tar files.
make KUBE_FORMATS="tar" master node
make KUBE_FORMATS="iso-bios" master node
mv kube-master*.iso kube-node*.iso /workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK
- persist_to_workspace:
root: /workspace
paths: images

image_boot: &image_boot
#machine:
# enabled: true
# image: circleci/classic:latest # default Ubuntu 14.04 image
macos:
xcode: "9.0"
steps:
- run:
name: Configure $PATH
#command: echo 'export PATH=~/workspace/bin-Linux:$PATH' >> $BASH_ENV
command: echo 'export PATH=~/workspace/bin-Darwin:$PATH' >> $BASH_ENV
- run:
name: Install packages
# ca-certificates are needed for attach_workspace (and git over https)
#command: sudo apt-get update && sudo apt-get install -y ca-certificates curl expect git make openssh-client procps qemu xorriso
command: brew install xorriso
- attach_workspace:
at: ~/workspace
- checkout
- run:
name: Versions
command: |
set -x
~/workspace/bin-Darwin/linuxkit version
~/workspace/bin-Darwin/vpnkit version
- run:
name: Test boot
command: |
set -x
# Add ssh keys to the image.
# TODO could this be integrated into linuxkit/mkimage-iso-*
# such that all those options do not need to be repeated
# here? e.g. perhaps this could be made to work:
# - Create tar with /root/.ssh/authorized_keys=~/workspace/ssh/id_rsa.pub in it
# - cat $tar | docker run -v ~/workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso:/input.iso linuxkit/mkimage-iso-bios -add-to /input.iso
#
# Compared with the options used i nmkimage-iso-bios to
# build the original dropped -J -joliet-long since xorriso
# is more particular about it than genisoimage producing
# lots of:
#
# Cannot add /bin/ls to Joliet tree. Symlinks can only be added to a Rock Ridge tree.

id_rsa=~/workspace/ssh/id_rsa.pub
xorriso -indev ~/workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso \
-outdev kube-master.iso \
-as genisoimage \
-l -R \
-c isolinux/boot.cat \
-b isolinux/isolinux.bin \
-no-emul-boot \
-boot-load-size 4 \
-boot-info-table \
-input-charset utf8 \
-- \
-pathspecs on \
-add /root/.ssh/authorized_keys=$id_rsa

#cp ~/workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-{master,node}.iso .
#cp ~/workspace/images/kube-$KUBE_RUNTIME-$KUBE_NETWORK/kube-master.iso .
SSHOPTS="-i ~/workspace/ssh/id_rsa" ./test.exp

version: 2
jobs:
Expand All @@ -95,48 +159,67 @@ jobs:
steps:
- run:
name: Create workspace
command: mkdir -p /workspace/bin
command: mkdir -p /workspace/bin-Linux /workspace/bin-Darwin
- run:
name: Install packages
command: apt-get update && apt-get install -y ca-certificates curl
command: apt-get update && apt-get install -y ca-certificates curl openssh-client
- run:
name: Fetch binaries
command: |
set -x
curl -fsSL -o /tmp/docker.tgz https://download.docker.com/linux/static/stable/x86_64/docker-17.06.2-ce.tgz
tar xfO /tmp/docker.tgz docker/docker > /workspace/bin/docker
curl -fsSL -o /workspace/bin/linuxkit https://362-46932243-gh.circle-artifacts.com/0/linuxkit-linux-amd64
curl -fsSL -o /workspace/bin/manifest-tool https://github.com/estesp/manifest-tool/releases/download/v0.7.0/manifest-tool-linux-amd64
curl -fsSL -o /workspace/bin/notary https://github.com/theupdateframework/notary/releases/download/v0.4.3/notary-Linux-amd64
tar xfO /tmp/docker.tgz docker/docker > /workspace/bin-Linux/docker
curl -fsSL -o /workspace/bin-Linux/linuxkit https://362-46932243-gh.circle-artifacts.com/0/linuxkit-linux-amd64
curl -fsSL -o /workspace/bin-Darwin/linuxkit https://362-46932243-gh.circle-artifacts.com/0/linuxkit-darwin-amd64
curl -fsSL -o /workspace/bin-Linux/manifest-tool https://github.com/estesp/manifest-tool/releases/download/v0.7.0/manifest-tool-linux-amd64
curl -fsSL -o /workspace/bin-Linux/notary https://github.com/theupdateframework/notary/releases/download/v0.4.3/notary-Linux-amd64

curl -fsSL -o /workspace/bin-Darwin/hyperkit https://452-55985023-gh.circle-artifacts.com/0/Users/distiller/hyperkit/build/hyperkit
curl -fsSL -o /tmp/vpnkit.tgz https://1066-58395340-gh.circle-artifacts.com/0/Users/distiller/vpnkit/vpnkit.tgz
tar xfO /tmp/vpnkit.tgz Contents/Resources/bin/vpnkit > /workspace/bin-Darwin/vpnkit

echo "Downloaded:"
sha256sum /workspace/bin/*
sha256sum /workspace/bin-Linux/* /workspace/bin-Darwin/*
echo

echo "Checking checksums"
sha256sum -c <<EOF
6af40e74b2dbb2927882acab52d50bfc72551779d541957fc70b6adc325ee5ef /workspace/bin/docker
b61188823c2491e9a1c16d275e41c68d5736f3f10577d01cb358261ff71bbad6 /workspace/bin/linuxkit
e4ca2ef0015a4be8597d31d9e3e70d88da33924ae72b0999e9f3b79304d4710d /workspace/bin/manifest-tool
06cd02c4c2e7a3b1ad9899b03b3d4dde5392d964c675247d32f604a24661f839 /workspace/bin/notary
6af40e74b2dbb2927882acab52d50bfc72551779d541957fc70b6adc325ee5ef /workspace/bin-Linux/docker
b61188823c2491e9a1c16d275e41c68d5736f3f10577d01cb358261ff71bbad6 /workspace/bin-Linux/linuxkit
908fae1c48122a245ddef8821a530a4c504f267ff1718b0c3688ef6f7aa3b65f /workspace/bin-Darwin/linuxkit
e4ca2ef0015a4be8597d31d9e3e70d88da33924ae72b0999e9f3b79304d4710d /workspace/bin-Linux/manifest-tool
06cd02c4c2e7a3b1ad9899b03b3d4dde5392d964c675247d32f604a24661f839 /workspace/bin-Linux/notary
298f650c9dd3ab8907b23f56cb21e33b794a5a8be163bbbcc7cf959112888b80 /workspace/bin-Darwin/hyperkit
72e34d9c0bc0a73bfb71f2b572849db5829ca41c3995245145f2e57c8ab40bd2 /workspace/bin-Darwin/vpnkit
EOF
- run:
name: Generate SSH Keys
command: |
mkdir -p /workspace/ssh
ssh-keygen -f /workspace/ssh/id_rsa -C "ephemeral CI ssh key" -N ''
- run:
name: Versions
command: |
chmod +x /workspace/bin/docker # docker version deferred until daemon configured in relevant jobs
chmod +x /workspace/bin/linuxkit && /workspace/bin/linuxkit version
chmod +x /workspace/bin/manifest-tool && /workspace/bin/manifest-tool --version
chmod +x /workspace/bin/notary && /workspace/bin/notary version
chmod +x /workspace/bin-Linux/docker # docker version deferred until daemon configured in relevant jobs
chmod +x /workspace/bin-Linux/linuxkit && /workspace/bin-Linux/linuxkit version
chmod +x /workspace/bin-Darwin/linuxkit # && /workspace/bin-Darwin/linuxkit version
chmod +x /workspace/bin-Linux/manifest-tool && /workspace/bin-Linux/manifest-tool --version
chmod +x /workspace/bin-Linux/notary && /workspace/bin-Linux/notary version
chmod +x /workspace/bin-Darwin/vpnkit # && /workspace/bin-Darwin/vpnkit version
- persist_to_workspace:
root: /workspace
paths: bin
paths:
- bin-Linux
- bin-Darwin
- ssh

lint:
docker:
- image: debian:stretch
steps:
- run:
name: Configure $PATH
command: echo 'export PATH=/workspace/bin:$PATH' >> $BASH_ENV
command: echo 'export PATH=/workspace/bin-Linux:$PATH' >> $BASH_ENV
- run:
name: Install packages
# ca-certificates are needed for attach_workspace (and git over https)
Expand Down Expand Up @@ -196,36 +279,60 @@ jobs:
<<: *linuxkit_pkg_build
pkg-cri-containerd:
<<: *linuxkit_pkg_build
pkg-kube-e2e-test:
<<: *linuxkit_pkg_build
#pkg-kube-e2e-test:
# <<: *linuxkit_pkg_build

pkg-kubernetes-docker-image-cache-common:
<<: *linuxkit_pkg_build
pkg-kubernetes-docker-image-cache-control-plane:
<<: *linuxkit_pkg_build
#pkg-kubernetes-docker-image-cache-common:
# <<: *linuxkit_pkg_build
#pkg-kubernetes-docker-image-cache-control-plane:
# <<: *linuxkit_pkg_build

image-docker-weave:
<<: *image_build
# Needs to be configured/enabled by CircleCI person
#resource_class: large
environment:
- KUBE_RUNTIME: docker
- KUBE_NETWORK: weave
image-docker-bridge:
<<: *image_build
# Needs to be configured/enabled by CircleCI person
#resource_class: large
environment:
- KUBE_RUNTIME: docker
- KUBE_NETWORK: bridge
#image-docker-weave:
# <<: *image_build
# # Needs to be configured/enabled by CircleCI person
# #resource_class: large
# environment:
# - KUBE_RUNTIME: docker
# - KUBE_NETWORK: weave
#image-docker-bridge:
# <<: *image_build
# # Needs to be configured/enabled by CircleCI person
# #resource_class: large
# environment:
# - KUBE_RUNTIME: docker
# - KUBE_NETWORK: bridge

image-cri-containerd-weave:
#image-cri-containerd-weave:
# <<: *image_build
# environment:
# - KUBE_RUNTIME: cri-containerd
# - KUBE_NETWORK: weave
image-cri-containerd-bridge:
<<: *image_build
environment:
- KUBE_RUNTIME: cri-containerd
- KUBE_NETWORK: weave
image-cri-containerd-bridge:
<<: *image_build
- KUBE_NETWORK: bridge

#boot-docker-weave:
# <<: *image_boot
# environment:
# - KUBE_RUNTIME: docker
# - KUBE_NETWORK: weave

#boot-docker-bridge:
# <<: *image_boot
# environment:
# - KUBE_RUNTIME: docker
# - KUBE_NETWORK: bridge

#boot-cri-containerd-weave:
# <<: *image_boot
# environment:
# - KUBE_RUNTIME: cri-containerd
# - KUBE_NETWORK: weave

boot-cri-containerd-bridge:
<<: *image_boot
environment:
- KUBE_RUNTIME: cri-containerd
- KUBE_NETWORK: bridge
Expand All @@ -236,7 +343,7 @@ jobs:
steps:
- run:
name: Configure $PATH
command: echo 'export PATH=/workspace/bin:$PATH' >> $BASH_ENV
command: echo 'export PATH=/workspace/bin-Linux:$PATH' >> $BASH_ENV
- run:
name: Install packages
# ca-certificates are needed for attach_workspace (and git over https)
Expand Down Expand Up @@ -268,6 +375,8 @@ jobs:
exit 0
fi

false # Debug branch, not expected to get this far

docker login -u $DOCKER_USER -p $DOCKER_PASS
mkdir -p ~/.docker/trust/private
cp .circleci/content-trust.key ~/.docker/trust/private/b056f84873aa0be205dfe826afa6e7458120c9569dd19a2a84154498fb1165d5.key
Expand All @@ -293,39 +402,55 @@ workflows:
- pkg-cri-containerd:
requires:
- dependencies
- pkg-kube-e2e-test:
requires:
- dependencies
- pkg-kubernetes-docker-image-cache-common:
requires:
- dependencies
- pkg-kubernetes-docker-image-cache-control-plane:
requires:
- dependencies
#- pkg-kube-e2e-test:
# requires:
# - dependencies
#- pkg-kubernetes-docker-image-cache-common:
# requires:
# - dependencies
#- pkg-kubernetes-docker-image-cache-control-plane:
# requires:
# - dependencies

- image-docker-weave:
requires:
- dependencies
- pkg-kubelet
- pkg-kubernetes-docker-image-cache-common
- pkg-kubernetes-docker-image-cache-control-plane
- image-docker-bridge:
requires:
- dependencies
- pkg-kubelet
- pkg-kubernetes-docker-image-cache-common
- pkg-kubernetes-docker-image-cache-control-plane
- image-cri-containerd-weave:
requires:
- dependencies
- pkg-kubelet
- pkg-cri-containerd
#- image-docker-weave:
# requires:
# - dependencies
# - pkg-kubelet
# - pkg-kubernetes-docker-image-cache-common
# - pkg-kubernetes-docker-image-cache-control-plane
#- image-docker-bridge:
# requires:
# - dependencies
# - pkg-kubelet
# - pkg-kubernetes-docker-image-cache-common
# - pkg-kubernetes-docker-image-cache-control-plane
#- image-cri-containerd-weave:
# requires:
# - dependencies
# - pkg-kubelet
# - pkg-cri-containerd
- image-cri-containerd-bridge:
requires:
- dependencies
- pkg-kubelet
- pkg-cri-containerd

#- boot-docker-weave:
# requires:
# - image-docker-weave

#- boot-docker-bridge:
# requires:
# - image-docker-bridge

#- boot-cri-containerd-weave:
# requires:
# - image-cri-containerd-weave

- boot-cri-containerd-bridge:
requires:
- image-cri-containerd-bridge

- push-pkgs-to-hub:
# We want everything to have passed, which is a bit
# tedious. Some of these are already covered transitively,
Expand All @@ -334,10 +459,14 @@ workflows:
- lint
- pkg-kubelet
- pkg-cri-containerd
- pkg-kube-e2e-test
- pkg-kubernetes-docker-image-cache-common
- pkg-kubernetes-docker-image-cache-control-plane
- image-docker-weave
- image-docker-bridge
- image-cri-containerd-weave
#- pkg-kube-e2e-test
#- pkg-kubernetes-docker-image-cache-common
#- pkg-kubernetes-docker-image-cache-control-plane
#- image-docker-weave
#- image-docker-bridge
#- image-cri-containerd-weave
- image-cri-containerd-bridge
#- boot-docker-weave
#- boot-docker-bridge
#- boot-cri-containerd-weave
- boot-cri-containerd-bridge
2 changes: 2 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ KUBE_FORMAT_ARGS := $(patsubst %,-format %,$(KUBE_FORMATS))
all: master node

master: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_RUNTIME)-master.yml yml/$(KUBE_NETWORK).yml
# strace logs received signals
#strace -f -e trace=signal
linuxkit $(LINUXKIT_ARGS) build $(LINUXKIT_BUILD_ARGS) -name kube-master $(KUBE_FORMAT_ARGS) $^

node: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_NETWORK).yml
Expand Down
Loading