Wait for eventual consistency on AWS spot instances

Also requesting a spot instance is subject to eventual consistency,
this patch implements a retry method around spot instance waiter.

Signed-off-by: Alessio Caiazza <acaiazza@gitlab.com>
This commit is contained in:
Alessio Caiazza
2017-11-16 18:55:21 +01:00
parent 6f126720e3
commit 28433ea85d
+17 -6
View File
@@ -45,7 +45,8 @@ const (
)
const (
keypairNotFoundCode = "InvalidKeyPair.NotFound"
keypairNotFoundCode = "InvalidKeyPair.NotFound"
spotInstanceRequestNotFoundCode = "InvalidSpotInstanceRequestID.NotFound"
)
var (
@@ -640,11 +641,21 @@ func (d *Driver) Create() error {
}
log.Info("Waiting for spot instance...")
err = d.getClient().WaitUntilSpotInstanceRequestFulfilled(&ec2.DescribeSpotInstanceRequestsInput{
SpotInstanceRequestIds: []*string{spotInstanceRequest.SpotInstanceRequests[0].SpotInstanceRequestId},
})
if err != nil {
return fmt.Errorf("Error fulfilling spot request: %v", err)
for i := 0; i < 3; i++ {
// AWS eventual consistency means we could not have SpotInstanceRequest ready yet
err = d.getClient().WaitUntilSpotInstanceRequestFulfilled(&ec2.DescribeSpotInstanceRequestsInput{
SpotInstanceRequestIds: []*string{spotInstanceRequest.SpotInstanceRequests[0].SpotInstanceRequestId},
})
if err != nil {
if awsErr, ok := err.(awserr.Error); ok {
if awsErr.Code() == spotInstanceRequestNotFoundCode {
time.Sleep(5 * time.Second)
continue
}
}
return fmt.Errorf("Error fulfilling spot request: %v", err)
}
break
}
log.Info("Created spot instance request %v", *spotInstanceRequest.SpotInstanceRequests[0].SpotInstanceRequestId)
// resolve instance id