Learn how to automate cloud infrastructure with ceph storage

# resource maps directly to services provided by cloud providers.
# it is always of the form x_y, wherein x is the cloud provider and y is the targeted service.
# the last part that follows is the name of the resource.
# below initializes attributes that are defined by the cloud provider to create VM.
resource "digitalocean_droplet" "admin" {
 image = "centos-7-0-x64"
 name = "ceph-admin"
 region = "sfo1"
 size = "1gb"
 private_networking = true
 ssh_keys = [
 "${var.ssh_fingerprint}"
 ]
# below defines the connection parameters necessary to do ssh for further customization.
# For this to work passwordless, the ssh keys should be pre-registered with cloud provider.
 connection {
 user = "root"
 type = "ssh"
 key_file = "${var.pvt_key}"
 timeout = "10m"
 }
# All below provisioners, perform the actual customization and run
# in the order specified in this file.
# "remote-exec" performs action on the remote VM over ssh.
# Below one could see some necessary directories are being created.
 provisioner "remote-exec" {
 inline = [
 "mkdir -p /opt/scripts /srv/salt /srv/pillar",
 "mkdir -p /srv/salt/users/cephadm/keys /srv/salt/users/demo/keys"',
 "mkdir -p /srv/salt/files",
 ]
 }
# "file" provisioner copies files from local workmachine (where the script is being run) to
# remote VM. Note the directories should exist, before this can pass.
# The below copies the whole directory contents from local machine to remote VM.
# These scripts help setup the whole environment and can be depended to be available at
# /opt/scripts location. Note, the scripts do not have executable permission bits set.
# Note the usage of "path.module", these are interpolation extensions provided by Terraform.
 provisioner "file" {
 source = "${path.module}/scripts/"
 destination = "/opt/scripts/"
 }
# A cephdeploy.repo file has to be made available at yum repo, for it to pick ceph packages.
# This requirement comes from setting up ceph storage cluster.
 provisioner "file" {
 source = "${path.module}/scripts/cephdeploy.repo"
 destination = "/etc/yum.repos.d/cephdeploy.repo"
 }
# Setup handcrafted custom sudoers file to allow running sudo through ssh without terminal connection.
# Also additionally provide necessary sudo permissions to cephadm user.
 provisioner "file" {
 source = "${path.module}/scripts/salt/salt/files/sudoers"
 destination = "/etc/sudoers"
 }
# Now, setup yum repos and install packages as necessary for Ceph admin node.
# Additionally ensure salt-master is installed.
# Create two users, cephadm privileged user with sudo access for managing the ceph cluster and demo guest user.
# The passwords are also set accordingly.
# Remember to set proper permissions to the scripts.
# The provisioned VM attributes can be easily used to customize several properties as needed. In our case,
# the IP address (public and private), VM host name are used to customize the environment further.
# For ex: hosts file, salt master configuration file and ssh_config file are updated accordingly.
 provisioner "remote-exec" {
 inline = [
 "export PATH=$PATH:/usr/bin",
 "chmod 0440 /etc/sudoers",
 "yum install -y epel-release yum-utils",
 "yum-config-manager --enable cr",
 "yum install -y yum-plugin-priorities",
 "yum clean all",
 "yum makecache",
 "yum install -y wget salt-master",
 "cp -af /opt/scripts/salt/* /srv",
 "yum install -y ceph-deploy --nogpgcheck",
 "yum install -y ntp ntpdate ntp-doc",
 "useradd -m -G wheel cephadm",
 "echo \"cephadm:c3ph@dm1n\" | chpasswd",
 "useradd -m -G docker demo",
 "echo \"demo:demo\" | chpasswd",
 "chmod +x /opt/scripts/*.sh",
 "/opt/scripts/fixadmin.sh ${self.ipv4_address} ${self.ipv4_address_private} ${self.name}",
 ]
 }
}

#!/bin/bash
# Expects ./fixadmin.sh
# Performs the following.
# a. caches cluster information in /opt/nodes
# b. patches /etc/hosts file to connect through private-ip for cluster communication.
# c. patches ssh_config file to enable auto connect without asking confirmation for given node.
# d. creates 2 users, with appropriate ssh keys
# e. customize salt configuration with cluster properties.
mkdir -p /opt/nodes
chmod 0755 /opt/nodes
echo "$1" > /opt/nodes/admin.public
echo "$2" > /opt/nodes/admin.private
rm -f /opt/nodes/masters*
sed -i '/demo-admin/d' /etc/hosts
echo "$2 demo-admin" >> /etc/hosts
sed -i '/demo-admin/,+1d' /etc/ssh/ssh_config
echo "Host demo-admin" >> /etc/ssh/ssh_config
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config
for user in cephadm demo; do
 rm -rf /home/${user}/.ssh
 su -c "cat /dev/zero | ssh-keygen -t rsa -N \"\" -q" ${user}
 cp /home/${user}/.ssh/id_rsa.pub /srv/salt/users/${user}/keys/key.pub
 cp /home/${user}/.ssh/id_rsa.pub /home/${user}/.ssh/authorized_keys
done
systemctl enable salt-master
systemctl stop salt-master
sed -i '/interface:/d' /etc/salt/master
echo "#script changes below" >> /etc/salt/master
echo "interface: ${2}" >> /etc/salt/master
systemctl start salt-master

[ceph-noarch]
name=Ceph noarch packages
baseurl=http://download.ceph.com/rpm-hammer/el7/noarch
enabled=1
gpgcheck=1
type=rpm-md
gpgkey=https://download.ceph.com/keys/release.asc

# resource specifies the attributes required to bring up ceph-monitor node.
# Note have the node name has been customized with an index, and the usage of 'count'
# 'count' is a special attribute that lets one create multiple instances of the same spec.
# That easy!
resource "digitalocean_droplet" "master" {
 image = "centos-7-0-x64"
 name = "ceph-monitor-${count.index}"
 region = "sfo1"
 size = "512mb"
 private_networking = true
 ssh_keys = [
 "${var.ssh_fingerprint}"
 ]
 count=1
 connection {
 user = "root"
 type = "ssh"
 key_file = "${var.pvt_key}"
 timeout = "10m"
 }
 provisioner "remote-exec" {
 inline = [
 "mkdir -p /opt/scripts /opt/nodes",
 ]
 }
 provisioner "file" {
 source = "${path.module}/scripts/"
 destination = "/opt/scripts/"
 }
 # This provisioner has implicit dependency on admin node to be available.
 # below we use admin node's property to fix ceph-monitor's salt minion configuration file,
 # so it can reach salt master.
 provisioner "remote-exec" {
 inline = [
 "export PATH=$PATH:/usr/bin",
 "yum install -y epel-release yum-utils",
 "yum-config-manager --enable cr",
 "yum install -y yum-plugin-priorities",
 "yum install -y salt-minion",
 "chmod +x /opt/scripts/*.sh",
 "/opt/scripts/fixsaltminion.sh ${digitalocean_droplet.admin.ipv4_address_private} ${self.name}",
 ]
 }
}

resource "digitalocean_droplet" "minion" {
    image = "centos-7-0-x64"
    name = "ceph-osd-${count.index}"
    region = "sfo1"
    size = "1gb"
    private_networking = true
    ssh_keys = [
      "${var.ssh_fingerprint}"
    ]
        # Here we specify two instances of this specification. Look above though the
        # hostnames are customized already by using interpolation.
    count=2
  connection {
      user = "root"
      type = "ssh"
      key_file = "${var.pvt_key}"
      timeout = "10m"
  }
  provisioner "remote-exec" {
    inline = [
      "mkdir -p /opt/scripts /opt/nodes",
    ]
  }
  provisioner "file" {
     source = "${path.module}/scripts/"
     destination = "/opt/scripts/"
  }
  provisioner "remote-exec" {
    inline = [
      "export PATH=$PATH:/usr/bin",
      "yum install -y epel-release yum-utils yum-plugin-priorities",
      "yum install -y salt-minion",
      "chmod +x /opt/scripts/*.sh",
      "/opt/scripts/fixsaltminion.sh ${digitalocean_droplet.admin.ipv4_address_private} ${self.name}",
    ]
  }
}

#!/bin/bash
# The below script ensures salt-minion nodes configuration file
# are patched to reach Salt master.
# args:
systemctl enable salt-minion
systemctl stop salt-minion
sed -i -e '/master:/d' /etc/salt/minion
echo "#scripted below config changes" >> /etc/salt/minion
echo "master: ${1}" >> /etc/salt/minion
echo "${2}" > /etc/salt/minion_id
systemctl start salt-minion

resource "null_resource" "cluster-init" {
    # so far we have relied on implicit dependency chain without specifying one.
        # Here we will ensure that this resources gets run only after successful creation of its
        # dependencies.
    depends_on = [
        "digitalocean_droplet.admin",
        "digitalocean_droplet.master",
        "digitalocean_droplet.minion",
    ]
  connection {
      host = "${digitalocean_droplet.admin.ipv4_address}"
      user = "root"
      type = "ssh"
      key_file = "${var.pvt_key}"
      timeout = "10m"
  }
  # Below we run few other scripts based on the cluster configuration.
    # And finally ensure all the other nodes in the cluster are ready for
    # ceph installation.
  provisioner "remote-exec" {
    inline = [
        "/opt/scripts/fixmasters.sh ${join(\" \", digitalocean_droplet.master.*.ipv4_address_private)}",
        "/opt/scripts/fixslaves.sh ${join(\" \", digitalocean_droplet.minion.*.ipv4_address_private)}",
        "salt-key -Ay",
        "salt -t 10 '*' test.ping",
        "salt -t 20 '*' state.apply common",
        "salt-cp '*' /opt/nodes/* /opt/nodes",
        "su -c /opt/scripts/ceph-pkgsetup.sh cephadm",
    ]
  }
}

#!/bin/bash
# This script fixes host file and collects cluster info under /opt/nodes
# Also updates ssh_config accordingly to ensure passwordless ssh can happen to
# other nodes in the cluster without prompting for confirmation.
# args:
NODES=""
i=0
for ip in "$@"
do
    NODE="ceph-monitor-$i"
    sed -i "/$NODE/d" /etc/hosts
    echo "$ip $NODE" >> /etc/hosts
    echo $NODE >> /opt/nodes/masters
    echo "$ip" >> /opt/nodes/masters.ip
    sed -i "/$NODE/,+1d" /etc/ssh/ssh_config
    NODES="$NODES $NODE"
    i=$[i+1]
done
echo "Host $NODES" >> /etc/ssh/ssh_config
echo "  StrictHostKeyChecking no" >> /etc/ssh/ssh_config

#!/bin/bash
# This script fixes host file and collects cluster info under /opt/nodes
# Also updates ssh_config accordingly to ensure passwordless ssh can happen to
# other nodes in the cluster without prompting for confirmation.
# args:
NODES=""
i=0
mkdir -p /opt/nodes
chmod 0755 /opt/nodes
rm -f /opt/nodes/minions*
for ip in "$@"
do
    NODE="ceph-osd-$i"
    sed -i "/$NODE/d" /etc/hosts
    echo "$ip $NODE" >> /etc/hosts
    echo $NODE >> /opt/nodes/minions
    echo "$ip" >> /opt/nodes/minions.ip
    sed -i "/$NODE/,+1d" /etc/ssh/ssh_config
    NODES="$NODES $NODE"
    i=$[i+1]
done
echo "Host $NODES" >> /etc/ssh/ssh_config
echo "  StrictHostKeyChecking no" >> /etc/ssh/ssh_config

#!/bin/bash
# has to be run as user 'cephadm' with sudo privileges.
# install ceph packages on all nodes in the cluster.
mkdir -p $HOME/my-cluster
cd $HOME/my-cluster
OPTIONS="--username cephadm --overwrite-conf"
echo "Installing ceph components"
RELEASE=hammer
for node in `sudo cat /opt/nodes/masters`
do
    ceph-deploy $OPTIONS install --release ${RELEASE} $node
done
for node in `sudo cat /opt/nodes/minions`
do
    ceph-deploy $OPTIONS install --release ${RELEASE} $node
done

resource "null_resource" "cluster-bootstrap" {
    depends_on = [
        "null_resource.cluster-init",
    ]
  connection {
      host = "${digitalocean_droplet.admin.ipv4_address}"
      user = "root"
      type = "ssh"
      key_file = "${var.pvt_key}"
      timeout = "10m"
  }
  provisioner "remote-exec" {
    inline = [
        "su -c /opt/scripts/ceph-install.sh cephadm",
        "salt 'ceph-monitor-*' state.highstate",
        "salt 'ceph-osd-*' state.highstate",
    ]
  }
}

#!/bin/bash
# This script has to be run as user 'cephadm', because  this user has
# sudo privileges set all across the cluster.
OPTIONS="--username cephadm --overwrite-conf"
# pre-cleanup.
rm -rf $HOME/my-cluster
for node in `cat /opt/nodes/masters /opt/nodes/minions`
do
    ssh $node "sudo rm -rf /etc/ceph/* /var/local/osd* /var/lib/ceph/mon/*"
    ssh $node "find /var/lib/ceph -type f | xargs sudo rm -rf"
done
mkdir -p $HOME/my-cluster
cd $HOME/my-cluster
echo "1. Preparing for ceph deployment"
ceph-deploy $OPTIONS new ceph-monitor-0
# Adjust the configuration to suit our cluster.
echo "osd pool default size = 2" >> ceph.conf
echo "osd pool default pg num = 16" >> ceph.conf
echo "osd pool default pgp num = 16" >> ceph.conf
echo "public network = `cat /opt/nodes/admin.private`/16" >> ceph.conf
echo "2. Add monitor and gather the keys"
ceph-deploy $OPTIONS mon create-initial
echo "3. Create OSD directory on each minions"
i=0
OSD=""
for node in `cat /opt/nodes/minions`
do
    ssh $node sudo mkdir -p /var/local/osd$i
    ssh $node sudo chown -R cephadm:cephadm /var/local/osd$i
    OSD="$OSD $node:/var/local/osd$i"
    i=$[i+1]
done
echo "4. Prepare OSD on minions - $OSD"
ceph-deploy $OPTIONS osd prepare $OSD
echo "5. Activate OSD on minions"
ceph-deploy $OPTIONS osd activate $OSD
echo "6. Copy keys to all nodes"
for node in `cat /opt/nodes/masters`
do
    ceph-deploy $OPTIONS admin $node
done
for node in `cat /opt/nodes/minions`
do
    ceph-deploy $OPTIONS admin $node
done
echo "7. Set permission on keyring"
sudo chmod +r /etc/ceph/ceph.client.admin.keyring
echo "8. Add in more monitors in cluster if available"
for mon in `cat /opt/nodes/masters`
do
    if [ "$mon" != "ceph-monitor-0" ]; then
        ceph-deploy $OPTIONS mon create $mon
    fi
done

# file: top.sls
base:
  "*":
    - users

# file: users.sls
groups:
users:
  cephadm:
    fullname: cephadm
    uid: 5000
    gid: 5000
    shell: /bin/bash
    home: /home/cephadm
    groups:
      - wheel
    password: $6$zYFWr3Ouemhtbnxi$kMowKkBYSh8tt2WY98whRcq.
    enforce_password: True
    key.pub: True
  demo:
    fullname: demo
    uid: 5031
    gid: 5031
    shell: /bin/bash
    home: /home/demo
    password: $6$XmIJ.Ox4tNKHa4oYccsYOEszswy1
    key.pub: True

# file: top.sls
base:
    "*":
        - common
    "ceph-admin":
        - admin
    "ceph-monitor-*":
        - master
    "ceph-osd-*":
        - minion

# file: common.sls
{% for group, args in pillar['groups'].iteritems() %}
{{ group }}:
  group.present:
    - name: {{ group }}
{% if 'gid' in args %}
    - gid: {{ args['gid'] }}
{% endif %}
{% endfor %}
{% for user, args in pillar['users'].iteritems() %}
{{ user }}:
  group.present:
    - gid: {{ args['gid'] }}
  user.present:
    - home: {{ args['home'] }}
    - shell: {{ args['shell'] }}
    - uid: {{ args['uid'] }}
    - gid: {{ args['gid'] }}
{% if 'password' in args %}
    - password: {{ args['password'] }}
{% if 'enforce_password' in args %}
    - enforce_password: {{ args['enforce_password'] }}
{% endif %}
{% endif %}
    - fullname: {{ args['fullname'] }}
{% if 'groups' in args %}
    - groups: {{ args['groups'] }}
{% endif %}
    - require:
      - group: {{ user }}
{% if 'key.pub' in args and args['key.pub'] == True %}
{{ user }}_key.pub:
  ssh_auth:
    - present
    - user: {{ user }}
    - source: salt://users/{{ user }}/keys/key.pub
  ssh_known_hosts:
    - present
    - user: {{ user }}
    - key: salt://users/{{ user }}/keys/key.pub
    - name: "demo-master-0"
{% endif %}
{% endfor %}
/etc/sudoers:
  file.managed:
    - source: salt://files/sudoers
    - user: root
    - group: root
    - mode: 440
/opt/nodes:
  file.directory:
    - user: root
    - group: root
    - mode: 755
/opt/scripts:
  file.directory:
    - user: root
    - group: root
    - mode: 755

# file: admin.sls
include:
  - master
bash /opt/scripts/bootstrap.sh:
  cmd.run

# file: master.sls
# one can include any packages, configuration to target ceph monitor nodes here.
masterpkgs:
    pkg.installed:
    - pkgs:
      - ntp
      - ntpdate
      - ntp-doc

# file: minion.sls
# one can include any packages, configuration to target ceph osd nodes here.
minionpkgs:
    pkg.installed:
    - pkgs:
      - ntp
      - ntpdate
      - ntp-doc

# file: files/sudoers
# customized for setting up environment to satisfy
# ceph pre-flight checks.
#
## Sudoers allows particular users to run various commands as
## the root user, without needing the root password.
##
## Examples are provided at the bottom of the file for collections
## of related commands, which can then be delegated out to particular
## users or groups.
##
## This file must be edited with the 'visudo' command.
## Host Aliases
## Groups of machines. You may prefer to use hostnames (perhaps using
## wildcards for entire domains) or IP addresses instead.
# Host_Alias     FILESERVERS = fs1, fs2
# Host_Alias     MAILSERVERS = smtp, smtp2
## User Aliases
## These aren't often necessary, as you can use regular groups
## (ie, from files, LDAP, NIS, etc) in this file - just use %groupname
## rather than USERALIAS
# User_Alias ADMINS = jsmith, mikem
## Command Aliases
## These are groups of related commands...
## Networking
# Cmnd_Alias NETWORKING = /sbin/route, /sbin/ifconfig, /bin/ping, /sbin/dhclient, /usr/bin/net, /sbin/iptables, /usr/bin/rfcomm, /usr/bin/wvdial, /sbin/iwconfig, /sbin/mii-tool
## Installation and management of software
# Cmnd_Alias SOFTWARE = /bin/rpm, /usr/bin/up2date, /usr/bin/yum
## Services
# Cmnd_Alias SERVICES = /sbin/service, /sbin/chkconfig
## Updating the locate database
# Cmnd_Alias LOCATE = /usr/bin/updatedb
## Storage
# Cmnd_Alias STORAGE = /sbin/fdisk, /sbin/sfdisk, /sbin/parted, /sbin/partprobe, /bin/mount, /bin/umount
## Delegating permissions
# Cmnd_Alias DELEGATING = /usr/sbin/visudo, /bin/chown, /bin/chmod, /bin/chgrp
## Processes
# Cmnd_Alias PROCESSES = /bin/nice, /bin/kill, /usr/bin/kill, /usr/bin/killall
## Drivers
# Cmnd_Alias DRIVERS = /sbin/modprobe
# Defaults specification
#
# Disable "ssh hostname sudo ", because it will show the password in clear.
#         You have to run "ssh -t hostname sudo ".
#
Defaults:cephadm    !requiretty
#
# Refuse to run if unable to disable echo on the tty. This setting should also be
# changed in order to be able to use sudo without a tty. See requiretty above.
#
Defaults   !visiblepw
#
# Preserving HOME has security implications since many programs
# use it when searching for configuration files. Note that HOME
# is already set when the the env_reset option is enabled, so
# this option is only effective for configurations where either
# env_reset is disabled or HOME is present in the env_keep list.
#
Defaults    always_set_home
Defaults    env_reset
Defaults    env_keep =  "COLORS DISPLAY HOSTNAME HISTSIZE INPUTRC KDEDIR LS_COLORS"
Defaults    env_keep += "MAIL PS1 PS2 QTDIR USERNAME LANG LC_ADDRESS LC_CTYPE"
Defaults    env_keep += "LC_COLLATE LC_IDENTIFICATION LC_MEASUREMENT LC_MESSAGES"
Defaults    env_keep += "LC_MONETARY LC_NAME LC_NUMERIC LC_PAPER LC_TELEPHONE"
Defaults    env_keep += "LC_TIME LC_ALL LANGUAGE LINGUAS _XKB_CHARSET XAUTHORITY"
#
# Adding HOME to env_keep may enable a user to run unrestricted
# commands via sudo.
#
# Defaults   env_keep += "HOME"
Defaults    secure_path = /sbin:/bin:/usr/sbin:/usr/bin
## Next comes the main part: which users can run what software on
## which machines (the sudoers file can be shared between multiple
## systems).
## Syntax:
##
##      user    MACHINE=COMMANDS
##
## The COMMANDS section may have other options added to it.
##
## Allow root to run any commands anywhere
root    ALL=(ALL)       ALL
## Allows members of the 'sys' group to run networking, software,
## service management apps and more.
# %sys ALL = NETWORKING, SOFTWARE, SERVICES, STORAGE, DELEGATING, PROCESSES, LOCATE, DRIVERS
## Allows people in group wheel to run all commands
# %wheel  ALL=(ALL)       ALL
## Same thing without a password
%wheel        ALL=(ALL)       NOPASSWD: ALL
## Allows members of the users group to mount and unmount the
## cdrom as root
# %users  ALL=/sbin/mount /mnt/cdrom, /sbin/umount /mnt/cdrom
## Allows members of the users group to shutdown this system
# %users  localhost=/sbin/shutdown -h now
## Read drop-in files from /etc/sudoers.d (the # here does not mean a comment)
#includedir /etc/sudoers.d