Skip to content

Commit c723c00

Browse files
ijcijc
authored
Merge pull request linuxkit#58 from ijc/add-rtf-boot-tests
Convenient local test harness for quick smoke test
2 parents 049cf07 + f4cd649 commit c723c00

File tree

11 files changed

+460
-4
lines changed

11 files changed

+460
-4
lines changed

‎Makefile

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,16 @@ endif
1111

1212
KUBE_FORMAT_ARGS := $(patsubst %,-format %,$(KUBE_FORMATS))
1313

14+
KUBE_BASENAME ?= kube-
15+
1416
.PHONY: all master node
1517
all: master node
1618

17-
master: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_RUNTIME)-master.yml yml/$(KUBE_NETWORK).yml
18-
linuxkit $(LINUXKIT_ARGS) build $(LINUXKIT_BUILD_ARGS) -name kube-master $(KUBE_FORMAT_ARGS) $^
19+
master: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_RUNTIME)-master.yml yml/$(KUBE_NETWORK).yml $(KUBE_EXTRA_YML)
20+
linuxkit $(LINUXKIT_ARGS) build $(LINUXKIT_BUILD_ARGS) -name $(KUBE_BASENAME)master $(KUBE_FORMAT_ARGS) $^
1921

20-
node: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_NETWORK).yml
21-
linuxkit $(LINUXKIT_ARGS) build $(LINUXKIT_BUILD_ARGS) -name kube-node $(KUBE_FORMAT_ARGS) $^
22+
node: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_NETWORK).yml $(KUBE_EXTRA_YML)
23+
linuxkit $(LINUXKIT_ARGS) build $(LINUXKIT_BUILD_ARGS) -name $(KUBE_BASENAME)node $(KUBE_FORMAT_ARGS) $^
2224

2325
yml/weave.yml: kube-weave.yaml
2426

‎test/.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
_results
2+
cases/_tmp

‎test/cases/000_smoke/001_cri-bridge/test.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/sh
2+
# SUMMARY: build and boot using cri-containerd runtime and Bridged networking
3+
# LABELS:
4+
5+
runtime=cri-containerd
6+
network=bridge
7+
8+
# Doesn't return
9+
. ../common.sh

‎test/cases/000_smoke/002_cri-weave/test.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/sh
2+
# SUMMARY: build and boot using cri-containerd runtime and Weave networking
3+
# LABELS:
4+
5+
runtime=cri-containerd
6+
network=weave
7+
8+
# Doesn't return
9+
. ../common.sh

‎test/cases/000_smoke/003_docker-bridge/test.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/sh
2+
# SUMMARY: build and boot using Docker runtime and Bridged networking
3+
# LABELS:
4+
5+
runtime=docker
6+
network=bridge
7+
8+
# Doesn't return
9+
. ../common.sh

‎test/cases/000_smoke/004_docker-weave/test.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/sh
2+
# SUMMARY: build and boot using Docker runtime and Weave networking
3+
# LABELS:
4+
5+
runtime=docker
6+
network=weave
7+
8+
# Doesn't return
9+
. ../common.sh

‎test/cases/000_smoke/common.sh

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# To be sourced by */test.sh
2+
3+
set -e
4+
5+
# Source libraries. Uncomment if needed/defined
6+
#. "${RT_LIB}"
7+
#. "${RT_PROJECT_ROOT}/_lib/lib.sh"
8+
9+
if [ "x$runtime" = "x" ] || [ "x$network" = "x" ] ; then
10+
echo "common.sh requires \$runtime and \$network" >&2
11+
exit 1
12+
fi
13+
14+
clean_up() {
15+
rm -f kube-master.iso
16+
rm -rf kube-master-state
17+
}
18+
19+
trap clean_up EXIT
20+
21+
export KUBE_RUNTIME=$runtime
22+
export KUBE_NETWORK=$network
23+
export LINUXKIT_BUILD_ARGS="--disable-content-trust"
24+
export KUBE_BASENAME="`pwd`/kube-"
25+
export KUBE_EXTRA_YML="`pwd`/../test.yml"
26+
make -C ${RT_PROJECT_ROOT}/../../ master
27+
28+
../test.exp ${RT_PROJECT_ROOT}/../../boot.sh ${RT_PROJECT_ROOT}/../../ssh_into_kubelet.sh
29+
30+
exit 0

‎test/cases/000_smoke/group.sh

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/sh
2+
# SUMMARY: Basic build and run smoke tests
3+
# LABELS:
4+
5+
group_init() {
6+
# Group initialisation code goes here
7+
return 0
8+
}
9+
10+
group_deinit() {
11+
# Group de-initialisation code goes here
12+
return 0
13+
}
14+
15+
CMD=$1
16+
case $CMD in
17+
init)
18+
group_init
19+
res=$?
20+
;;
21+
deinit)
22+
group_deinit
23+
res=$?
24+
;;
25+
*)
26+
res=1
27+
;;
28+
esac
29+
30+
exit $res
31+

‎test/cases/000_smoke/test.exp

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
#!/usr/bin/env expect
2+
3+
set con_prompt "(ns: getty) linuxkit-*:*# "
4+
set ssh_prompt "linuxkit-*:*# "
5+
set timeout 120
6+
7+
set bootexec [lindex $argv 0]
8+
set sshexec [lindex $argv 1]
9+
10+
proc kill args {
11+
foreach what $args {
12+
global $what
13+
if [info exists $what] {
14+
upvar #0 $what sid
15+
set pid [exp_pid -i $sid]
16+
puts "killing $what ($sid) = $pid"
17+
exec kill $pid
18+
close $sid
19+
} else {
20+
puts "not killing $what (not started)"
21+
}
22+
}
23+
}
24+
25+
proc boot_linuxkit {} {
26+
global lk_sid bootexec
27+
spawn env {KUBE_RUN_ARGS=-publish 2222:22 -publish 30002:30002} KUBE_MEM=3584 KUBE_VCPUS=2 KUBE_DISK=8G KUBE_CLEAR_STATE=y KUBE_MASTER_UNTAINT=y $bootexec
28+
set lk_sid $spawn_id
29+
puts "INFO lk ($lk_sid) is pid [exp_pid -i $lk_sid]"
30+
}
31+
32+
proc ssh_into_kubelet {} {
33+
global ssh_sid sshexec
34+
35+
set sshopts {-p 2222 -o ConnectTimeout=5 -o LogLevel=DEBUG}
36+
if [info exists ::env(SSHOPTS)] {
37+
set sshopts [concat $::env(SSHOPTS) $sshopts]
38+
}
39+
spawn env SSHOPTS=$sshopts $sshexec localhost
40+
set ssh_sid $spawn_id
41+
puts "INFO ssh ($ssh_sid) is pid [exp_pid -i $ssh_sid]"
42+
}
43+
44+
proc await_prompt {sidvar promptvar step} {
45+
upvar #0 $sidvar sid $promptvar prompt
46+
expect -i $sid \
47+
$prompt {
48+
puts "SUCCESS $step"
49+
} timeout {
50+
puts "FAIL $step (timeout)"
51+
kill ssh_sid lk_sid
52+
exit 1
53+
} eof {
54+
puts "FAIL $step (eof)"
55+
kill ssh_sid lk_sid
56+
exit 1
57+
}
58+
}
59+
60+
proc send_concmd {s} {
61+
global lk_sid
62+
send -i $lk_sid "$s\n"
63+
}
64+
65+
proc await_con_prompt {step} {
66+
global lk_sid con_prompt
67+
await_prompt lk_sid con_prompt $step
68+
}
69+
70+
proc concmd {step cmd} {
71+
send_concmd $cmd
72+
await_con_prompt $step
73+
}
74+
75+
proc send_sshcmd {s} {
76+
global ssh_sid
77+
send -i $ssh_sid "$s\n"
78+
}
79+
80+
proc await_ssh_prompt {step} {
81+
global ssh_sid ssh_prompt
82+
await_prompt ssh_sid ssh_prompt $step
83+
}
84+
85+
proc sshcmd {step cmd} {
86+
send_sshcmd $cmd
87+
await_ssh_prompt $step
88+
}
89+
90+
# Run $cmd at 1s intervals until we get $want or timeout
91+
# $sidvar names a variable containing sid to send/receive on
92+
# $promptvar names a variable containing the expected prompt
93+
# $step is used for logging
94+
proc await_cmd_output {sidvar promptvar step cmd want} {
95+
upvar #0 $sidvar sid $promptvar prompt
96+
send -i $sid "$cmd\n"
97+
set retries 0
98+
expect -i $sid -timeout 300 \
99+
$want {
100+
puts "SUCCESS $step: $cmd => $want"
101+
} $prompt {
102+
set retries [expr $retries + 1]
103+
sleep 1
104+
send -i $sid "$cmd\n"
105+
exp_continue -continue_timer
106+
} timeout {
107+
puts "FAIL $step (timeout)"
108+
kill ssh_sid lk_sid
109+
exit 1
110+
}
111+
await_prompt $sidvar $promptvar "$step (after $retries attempts)"
112+
}
113+
114+
proc await_sshcmd_output {step cmd want} {
115+
global ssh_sid ssh_prompt
116+
await_cmd_output ssh_sid ssh_prompt $step $cmd $want
117+
}
118+
119+
boot_linuxkit
120+
121+
await_con_prompt "boot"
122+
123+
concmd "ifconfig" "ifconfig eth0"
124+
concmd "list containers" "ctr -n services.linuxkit container ls"
125+
126+
set retries 0
127+
ssh_into_kubelet
128+
# provide ssh_sid as an indirect, allowing ssh to be respawned, which
129+
# changes the id, we need this in case ssh cannot immediately connect.
130+
expect -i ssh_sid \
131+
$ssh_prompt {
132+
puts "SUCCESS connected to ssh (after $retries attempts)"
133+
} "read: Connection reset by peer" {
134+
# ssh happened too soon, wait a bit.
135+
set retries [expr $retries + 1]
136+
#puts "RETRY:$retries ssh (conn reset)"
137+
wait -i $ssh_sid
138+
sleep 1
139+
ssh_into_kubelet
140+
exp_continue -continue_timer
141+
} eof {
142+
set retries [expr $retries + 1]
143+
#puts "RETRY:$retries ssh (eof)"
144+
wait -i $ssh_sid
145+
sleep 1
146+
ssh_into_kubelet
147+
exp_continue -continue_timer
148+
} timeout {
149+
puts "FAIL ssh (timeout)"
150+
kill ssh_sid lk_sid
151+
exit 1
152+
}
153+
154+
puts "RUN kubeadm-init.sh"
155+
send_sshcmd "kubeadm-init.sh"
156+
157+
# Written as N*5m with logging to avoid "10 mins with no output" from CI
158+
set retries 0
159+
set maxretries 10
160+
expect -i $ssh_sid -timeout 300 \
161+
"Your Kubernetes master has initialized successfully!" {
162+
puts "SUCCESS cluster initialised!"
163+
} $ssh_prompt {
164+
puts "FAIL kubeadm-init.sh (returned to prompt)"
165+
kill ssh_sid lk_sid
166+
exit 1
167+
} timeout {
168+
set retries [expr $retries + 1]
169+
if [expr $retries < $maxretries] {
170+
#puts "RETRY:$retries kubeadm-init.sh (timeout)"
171+
exp_continue
172+
}
173+
puts "FAIL kubeadm-init.sh (timeout)"
174+
kill ssh_sid lk_sid
175+
exit 1
176+
} eof {
177+
puts "FAIL kubeadm-init.sh (eof)"
178+
kill ssh_sid lk_sid
179+
exit 1
180+
}
181+
await_ssh_prompt "kubeadm-init.sh"
182+
183+
sshcmd "status" "kubectl get -o wide nodes ; echo ; kubectl --namespace=kube-system get -o wide pods"
184+
185+
await_sshcmd_output "await node ready" \
186+
{kubectl get nodes -o jsonpath='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' ; echo} \
187+
"Ready=True"
188+
189+
await_sshcmd_output "await kube-dns ready" \
190+
{kubectl --namespace=kube-system get --selector='k8s-app'='kube-dns' -o jsonpath='{.items[*].status.phase}' pods ; echo} \
191+
"Running"
192+
await_sshcmd_output "await etcd ready" \
193+
{kubectl --namespace=kube-system get --selector='component'='etcd' -o jsonpath='{.items[*].status.phase}' pods ; echo} \
194+
"Running"
195+
196+
sshcmd "status" "kubectl get -o wide nodes ; echo ; kubectl --namespace=kube-system get -o wide pods"
197+
198+
sshcmd "apply app.yaml" "kubectl apply -f /root/app.yaml"
199+
200+
await_sshcmd_output "await alpine pod ready" \
201+
{kubectl get pods --selector=name=alpine -o jsonpath='{.items[*].status.phase}' ; echo} \
202+
"Running"
203+
await_sshcmd_output "await nginx pod ready" \
204+
{kubectl get pods --selector=name=nginx -o jsonpath='{.items[*].status.phase}' ; echo} \
205+
"Running"
206+
puts "SUCCESS application pods ready"
207+
208+
sshcmd "status" "kubectl get -o wide nodes ; echo ; kubectl --namespace=kube-system get -o wide pods ; echo ; kubectl --namespace=default get -o wide pods"
209+
210+
set curl [exec curl -sS http://localhost:30002]
211+
if [string match "Welcome to nginx!" $curl] {
212+
puts "FAIL nginx returned:"
213+
puts $curl
214+
kill ssh_sid lk_sid
215+
exit 1
216+
}
217+
puts "SUCCESS nginx responded well"
218+
219+
send_sshcmd {kubectl exec $(kubectl get pods -l name==alpine -o=jsonpath='{.items[*].metadata.name}') -- wget -q -O - http://nginx/}
220+
expect -i $ssh_sid -timeout 10 \
221+
"Welcome to nginx!" {
222+
puts "SUCCESS intra-pod networking ok"
223+
} $ssh_prompt {
224+
puts "FAIL intra-pod networking (returned to prompt)"
225+
kill ssh_sid lk_sid
226+
exit 1
227+
} timeout {
228+
puts "FAIL intra-pod networking (timeout)"
229+
kill ssh_sid lk_sid
230+
exit 1
231+
} eof {
232+
puts "FAIL intra-pod networking (eof)"
233+
kill ssh_sid lk_sid
234+
exit 1
235+
}
236+
await_ssh_prompt "intra-pod networking"
237+
238+
kill ssh_sid
239+
240+
puts "RUN poweroff -f"
241+
send_concmd "poweroff -f"
242+
243+
expect -i $lk_sid \
244+
"Power down" {
245+
puts "SUCCESS poweroff"
246+
} eof {
247+
puts "SUCCESS poweroff"
248+
} timeout {
249+
puts "FAILED poweroff (timeout)"
250+
exit 1
251+
}

0 commit comments

Comments
 (0)