Scaling Automation

I’m slowly on my way to automating the creation of droplets in Digital Ocean and adding them to my docker swarm cluster. I have the creation and configuration working thanks to PowerShell and Ansible. I’m still working on the check logic but have the queries I need for InfluxDB. I need to figure out the best way to query the database. I also started thinking about ensuring there are limitations so this thing doesn’t take off and end up spawning 100+ servers overnight by mistake. I also am trying to set up Graylog to drive alerting for actions taken by this script and a general troubleshooting tool. I’m having trouble getting it to work inside Docker but I’ll work on it some more tomorrow. Here’s the work in progress PowerShell script. The first two functions work but there’s definitely improvements to be made.

#Requirements: Digital Ocean command line, powershell running on linux, ansible
#I can't seem to find a way to find droplets associated with a project so I'm kind of cheating and finding droplets with swarm in the name
Write-Host "Starting script to create and configure new swarm worker node" -ForegroundColor Blue
function Create-Droplet {
Write-Host "Retrieveing existing nodes from DigitalOcean"
$droplets = doctl compute droplet list --output json | ConvertFrom-Json
$swarmIDs = New-Object System.Collections.ArrayList
foreach($droplet in $droplets){
    if($droplet.name -like 'swarm*'){
    $swarmIDs.add($droplet.id)
    }
}

#If droplets get destroyed this should allow those numbers to get reused and not over increment
Write-Host "Finding next swarm node number"
$findIncrement = 1
foreach($swarmID in $swarmIDs){
    $swarmNode = doctl compute droplet get $($swarmID) --output json | ConvertFrom-Json
    if($swarmNode.name -like "*$($findIncrement)*"){
        $findIncrement++
    }
}
Write-Host "Found next available number is $($findIncrement)"
#Sets the name of the droplet to be created
$newName = "swarm$($findIncrement).serverhobbyist.net"
Write-Host "Name of node to be created: $($newName)"
#Checks for and deletes last json file for last created droplet to keep it current
$fileCheck = Test-Path -path /storage/ansible/nodescreated/latest.json
if ($fileCheck -eq $true){
    Remove-Item -Path /storage/ansible/nodescreated/latest.json
}
#Run Ansible Playbook to Create Droplet
$command = "ansible-playbook CreateDroplet.yaml -e `"dropletName=$($newName)`""
Write-Host "Running ansible playbook with command: $($command)"
bash -c $command
}
function Setup-Droplet {
#Gets returned data from Create-Droplet function
Write-Host "Discovering data about newly created node"
$createdNode = Get-Content  /storage/ansible/nodescreated/latest.json | ConvertFrom-Json
$IP = $createdNode.data.ip_address
#$IP = "165.22.178.216"
#Adds IP to hosts file
Write-Host "Writing IP $($IP) to Ansible Hosts file"
Add-Content /etc/ansible/hosts "`n$($IP)"
#Runs playbook to configure node: updates, telegraf agent + config, docker, epel, htop
$command = "ansible-playbook SetupDroplet.yaml -e `"IP=$($IP)`""
Write-Host "Running playbook with command: $($command)"
bash -c $command
}
function Check-Droplets {
#Finds current nodes in account with swarm in the name and adds the name to array
Write-Host "Retrieveing existing nodes from DigitalOcean"
$droplets = doctl compute droplet list --output json | ConvertFrom-Json
$swarmNames = New-Object System.Collections.ArrayList
foreach($droplet in $droplets){
    if($droplet.name -like 'swarm*'){
    $swarmNames.add($droplet.name)
    }
}

$check = 0
foreach($swarmName in $swarmNames){
    $influxCheckCPU = "q=`"select mean(usage_idle),count(usage_idle) from `"cpu`" where `"host`" = " + "'" + $($swarmName) + "' " + "and time > now() - 5m`""
    $influxCheckRAM = "q=`"select mean(used_percent),count(used_percent) from `"mem`" where `"host`" =" + "'" + $($swarmName) + "'" +" and time > now() - 5m`""
    $influxCPUResult = curl -G 'https://influx.server.com/query?pretty=true' -u user:pw --data-urlencode `"db=telegraf`" --data-urlencode $influxCheckCPU | ConvertFrom-Json
    $influxRAMResult = curl -G 'https://influx.server.com/query?pretty=true' -u user:pw --data-urlencode `"db=telegraf`" --data-urlencode $influxCheckRAM | ConvertFrom-Json
    if($influxCPUResult -ne $null -and $influxRAMResult -ne $null){
        $CPUUsage = (100 - $($influxCPUResult.results.series.values)[1])
        $RAMUsage  = $($influxRAMResult.results.series.values)[1]
        $CPUUsage = [math]::Round($($CPUUsage),2)
        $RAMUsage = [math]::Round($($RAMUsage),2)
        Write-Host "Results for $($swarmName):" -ForegroundColor Blue
        Write-host "CPU usage: $($CPUUsage)%" -ForegroundColor Blue
        Write-host "RAM usage: $($RAMUsage)%" -ForegroundColor Blue

        if($CPUUsage -ge 40){
            $CPUUsage = [math]::Round($($CPUUsage),2)
            Write-Host "CPU usage is now at $($CPUUsage)% which is high, an action might be taken"
            $command = "logger CPU usage is now at $($CPUUsage)% which is high, an action might be taken"
            $check++
            bash -c $command
            }
        if($RAMUsage -ge 50){
            $RAMUsage = [math]::Round($($RAMUsage),2)
            Write-Host "RAM usage is now at $($RAMUsage)% which is high, an action might be taken"
            $command = "logger RAM usage is now at $($RAMUsage)% which is high, an action might be taken"
            $check++
            bash -c $command
            }
        }
    }
}


#Write-Host "Starting Function: Create-Droplet" -ForegroundColor Green
#Create-Droplet
#Write-Host "Starting Function: Setup-Droplet" -ForegroundColor Green
#Setup-Droplet
Write-Host "Starting Function: Check-Droplet" -ForegroundColor Green
Check-Droplets