Working with Terraform
Note
For more background information, make sure to check out the demos overview page.
Terraform is great, check out the official docs.1
Managing multiple versions of terraform or using a dockerized version of terraform is pretty easy to do in a variety of ways, from a variety of execution contexts. As an automation framework, k8s.mk can do that too[^2] and avoid a terraform dependency on the host, which might be useful if you're just testing things out. But.. most likely if you're using terraform at all then you're also already comfortable with a tool like terragrunt[^3] or Atlantis[^4].
For power users, there are a few other use-cases that might be more interesting. This demo looks at two of the more unique possibilities with k8s.mk+terraform:
- Stateless operations,
- Where we avoid avoid additional directories and even files. This is kind of like fire-and-forget HCL and allows you to scriptify snippets, and quickly build tools or APIs. For example you can use this to expose data-resources to other tools, or quickly expose a CLI api to start jobs that are tracked by some other system. (See the first demo)
- Polyglots and hybrids,
- Easily mix terraform with other languages/systems, for hybrid or heterogenous deployment automation. Pass data between other tool containers, all in the same file, without any need to define external registries or repositories. (See the 2nd demo)
Source Code
#!/usr/bin/env -S make -f
# S3-Compatible object storage with minio.
#
# See the documentation here[1] for more discussion.
# This demo ships with the `k8s-tools` repo and runs as part of the test-suite.
#
# USAGE:
# ./demos/minio.mk clean create deploy test
#
# REF:
# [1] https://robot-wranglers.github.io/k8s-tools/demos/minio/
#░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░
include k8s.mk
export KUBECONFIG:=./local.cluster.yml
$(shell umask 066; touch ${KUBECONFIG})
$(call compose.import, file=k8s-tools.yml)
## Cluster lifecycle basics. These are the same for most demos.
cluster.name=terraform
minikube.args=--driver=docker -v1 --wait=all --embed-certs \
__main__: init clean create deploy test
clean: stage/clean minikube.delete/${cluster.name}
create: stage/create minikube.get_or_create/${cluster.name}
wait: k8s.pods.wait
test: terraform.test
deploy: stage/deploy infra.setup wait
##
infra.setup:
terraform.test:; $(call terraform_tasks)
define terraform.test
terraform {}
variable "KUBECONFIG" {
type = string
default = "~/.kube/config" # or leave empty
}
provider "kubernetes" {
config_path = var.KUBECONFIG
}
data "kubernetes_namespace" "kube_system" {
metadata { name = "kube-system" }
}
data "kubernetes_namespace" "default" {
metadata { name = "default" }
}
output "connection_test" {
description = "Basic connection test to verify Terraform can access the cluster"
value = {
kube_system_namespace = data.kubernetes_namespace.kube_system.metadata[0].name
default_namespace = data.kubernetes_namespace.default.metadata[0].name
connection_status = "SUCCESS - Terraform can access the cluster"
}
}
resource "random_id" "job_suffix" {
byte_length = 4
}
resource "kubernetes_job" "demo_job" {
metadata {
name = "terraform-demo-job-${random_id.job_suffix.hex}"
namespace = "default"
labels = {
app = "terraform-demo"
type = "job-demo"
managed = "terraform"
}
}
spec {
template {
metadata {
labels = { app = "terraform-demo-job" }
}
spec {
container {
name = "demo-job-container"
image = "busybox:latest"
command = [
"/bin/sh",
"-c",
"echo 'Hello from Terraform Kubernetes Job!'; echo 'Job started at:'; date; sleep 30; echo 'Job completed at:'; date"
]
}
restart_policy = "Never"
}
}
}
wait_for_completion = true
}
output "job_information" {
description = "Information about the created job"
value = {
job_name = kubernetes_job.demo_job.metadata[0].name
job_uid = kubernetes_job.demo_job.metadata[0].uid
}
}
endef
# # Output node capacity summary
# output "node_capacity_summary" {
# description = "Summary of node capacities"
# value = {
# for node in data.kubernetes_nodes.all_nodes.nodes : node.metadata[0].name => {
# cpu = node.status[0].capacity.cpu
# memory = node.status[0].capacity.memory
# pods = node.status[0].capacity.pods
# storage = try(node.status[0].capacity["ephemeral-storage"], "N/A")
# }
# }
# }
# # Output ready nodes only
# output "ready_nodes" {
# description = "List of nodes that are in Ready state"
# value = [
# for node in data.kubernetes_nodes.all_nodes.nodes : node.metadata[0].name
# if contains([
# for condition in node.status[0].conditions : condition.status
# if condition.type == "Ready"
# ], "True")
# ]
# }
#!/usr/bin/env -S make -f
# S3-Compatible object storage with minio.
#
# See the documentation here[1] for more discussion.
# This demo ships with the `k8s-tools` repo and runs as part of the test-suite.
#
# USAGE:
# ./demos/minio.mk clean create deploy test
#
# REF:
# [1] https://robot-wranglers.github.io/k8s-tools/demos/minio/
#░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░
include k8s.mk
export KUBECONFIG:=./local.cluster.yml
$(shell umask 066; touch ${KUBECONFIG})
$(call compose.import, file=k8s-tools.yml)
## Cluster lifecycle basics. These are the same for most demos.
cluster.name=terraform
minikube.args=--driver=docker -v1 --wait=all --embed-certs \
__main__: init clean create deploy test
clean: stage/clean minikube.delete/${cluster.name}
create: stage/create minikube.get_or_create/${cluster.name}
wait: k8s.pods.wait
test: terraform.test
deploy: stage/deploy infra.setup wait
##
infra.setup:
terraform.test:; $(call terraform_tasks)
define terraform.test
terraform {}
variable "KUBECONFIG" {
type = string
default = "~/.kube/config" # or leave empty
}
provider "kubernetes" {
config_path = var.KUBECONFIG
}
data "kubernetes_namespace" "kube_system" {
metadata { name = "kube-system" }
}
data "kubernetes_namespace" "default" {
metadata { name = "default" }
}
output "connection_test" {
description = "Basic connection test to verify Terraform can access the cluster"
value = {
kube_system_namespace = data.kubernetes_namespace.kube_system.metadata[0].name
default_namespace = data.kubernetes_namespace.default.metadata[0].name
connection_status = "SUCCESS - Terraform can access the cluster"
}
}
resource "random_id" "job_suffix" {
byte_length = 4
}
resource "kubernetes_job" "demo_job" {
metadata {
name = "terraform-demo-job-${random_id.job_suffix.hex}"
namespace = "default"
labels = {
app = "terraform-demo"
type = "job-demo"
managed = "terraform"
}
}
spec {
template {
metadata {
labels = { app = "terraform-demo-job" }
}
spec {
container {
name = "demo-job-container"
image = "busybox:latest"
command = [
"/bin/sh",
"-c",
"echo 'Hello from Terraform Kubernetes Job!'; echo 'Job started at:'; date; sleep 30; echo 'Job completed at:'; date"
]
}
restart_policy = "Never"
}
}
}
wait_for_completion = true
}
output "job_information" {
description = "Information about the created job"
value = {
job_name = kubernetes_job.demo_job.metadata[0].name
job_uid = kubernetes_job.demo_job.metadata[0].uid
}
}
endef
# # Output node capacity summary
# output "node_capacity_summary" {
# description = "Summary of node capacities"
# value = {
# for node in data.kubernetes_nodes.all_nodes.nodes : node.metadata[0].name => {
# cpu = node.status[0].capacity.cpu
# memory = node.status[0].capacity.memory
# pods = node.status[0].capacity.pods
# storage = try(node.status[0].capacity["ephemeral-storage"], "N/A")
# }
# }
# }
# # Output ready nodes only
# output "ready_nodes" {
# description = "List of nodes that are in Ready state"
# value = [
# for node in data.kubernetes_nodes.all_nodes.nodes : node.metadata[0].name
# if contains([
# for condition in node.status[0].conditions : condition.status
# if condition.type == "Ready"
# ], "True")
# ]
# }
Basic Usage
Cluster lifecycle looks more or less the same as any of the other demos.
# Default entrypoint runs clean, create,
# deploy, test, but does not tear down the cluster.
$ ./demos/terraform.mk clean create
# As part of `deploy`, `tilt up` will run.
$ ./demos/terraform.mk deploy
# In this case, test just retrieves logs for tilt,
# then drops a link for web UI so you can test.
$ ./demos/terraform.mk test
# Stop the daemonizes tilt-server, tear down the cluster.
$ ./demos/terraform.mk minio.stop clean