AWS Key Rotation Issues

Question

I am trying to follow the guide here to automate the rotation of keys for IAM users- https://awsfeed.com/whats-new/apn/automating-rotation-of-iam-user-access-and-secret-keys-with-aws-secrets-manager

Essentially I'm wanting to get new keys every 60 days, deactivate the old keys every 80 days, and then delete/remove old keys every 90 days.

I have slightly modified it to get new keys every 60 days instead of 90 and here is the lambda function:

import json
import boto3
import base64
import datetime
import os
from datetime import date
from botocore.exceptions import ClientError
iam = boto3.client('iam')
secretmanager = boto3.client('secretsmanager')
#IAM_UserName=os.environ['IAM_UserName']
#SecretName=os.environ['SecretName']

def create_key(uname):
    try:
        IAM_UserName=uname
        response = iam.create_access_key(UserName=IAM_UserName)
        AccessKey = response['AccessKey']['AccessKeyId']
        SecretKey = response['AccessKey']['SecretAccessKey']
        json_data=json.dumps({'AccessKey':AccessKey,'SecretKey':SecretKey})
        secmanagerv=secretmanager.put_secret_value(SecretId=IAM_UserName,SecretString=json_data)
        emailmsg="New "+AccessKey+" has been create. Please get the secret key value from secret manager"
        ops_sns_topic ='arn:aws:sns:us-east-1:redacted'
        sns_send_report = boto3.client('sns',region_name='us-east-1')
        sns_send_report.publish(TopicArn=ops_sns_topic, Message=emailmsg, Subject="New Key created for user"+ IAM_UserName)
    except ClientError as e:
        print (e)

def deactive_key(uname):
    try:
        #GET PREVIOUS AND CURRENT VERSION OF KEY FROM SECRET MANAGER
        IAM_UserName=uname
        getpresecvalue=secretmanager.get_secret_value(SecretId=IAM_UserName,VersionStage='AWSPREVIOUS')
        #getcursecvalue=secretmanager.get_secret_value(SecretId='secmanager3',VersionStage='AWSCURRENT')
        #print (getpresecvalue)
        #print (getcursecvalue)
        preSecString = json.loads(getpresecvalue['SecretString'])
        preAccKey=preSecString['AccessKey']
        #GET CREATION DATE OF CURRENT VERSION OF ACCESS KEY
        #curdate=getcursecvalue['CreatedDate']
        #GET TIMEZONE FROM CREATION DATE
        #tz=curdate.tzinfo
        #CALCULATE TIME DIFFERENCE BETWEEN CREATION DATE AND TODAY
        #diff=datetime.datetime.now(tz)-curdate
        #diffdays=diff.days
        #print (curdate)
        #print (tz)
        #print (diffdays)
        #print (preAccKey)
        #IF TIME DIFFERENCE IS MORE THAN x NUMBER OF DAYS THEN DEACTIVATE PREVIOUS KEY AND SEND A MESSAGE
        #if diffdays >= 1:
        iam.update_access_key(AccessKeyId=preAccKey,Status='Inactive',UserName=IAM_UserName)
        emailmsg="PreviousKey "+preAccKey+" has been disabled for IAM User"+IAM_UserName
        ops_sns_topic ='arn:aws:sns:us-east-1:redacted'
        sns_send_report = boto3.client('sns',region_name='us-east-1')
        sns_send_report.publish(TopicArn=ops_sns_topic, Message=emailmsg, Subject='Previous Key Deactivated')
        return
    except ClientError as e:
        print (e)
    #else:
    #    print ("Current Key is not older than 10 days")
    #print (datediff)

def delete_key(uname):
    try:
        IAM_UserName=uname
        print (IAM_UserName)
        getpresecvalue=secretmanager.get_secret_value(SecretId=IAM_UserName,VersionStage='AWSPREVIOUS')
    #getcursecvalue=secretmanager.get_secret_value(SecretId='secmanager3',VersionStage='AWSCURRENT')
        preSecString = json.loads(getpresecvalue['SecretString'])
        preAccKey=preSecString['AccessKey']
        #print (preAccKey)
    #GET CREATION DATE OF CURRENT VERSION OF ACCESS KEY
    #curdate=getcursecvalue['CreatedDate']
    #GET TIMEZONE FROM CREATION DATE
    #tz=curdate.tzinfo
    #CALCULATE TIME DIFFERENCE BETWEEN CREATION DATE AND TODAY
    #diff=datetime.datetime.now(tz)-curdate
    #diffdays=diff.days
    #IF TIME DIFFERENCE IS MORE THAN x NUMBER OF DAYS THEN DEACTIVATE PREVIOUS KEY AND SEND A MESSAGE
    #if diffdays >= 1:
        keylist=iam.list_access_keys (UserName=IAM_UserName)
    #print (keylist)
        for x in range(2):
            prevkeystatus=keylist['AccessKeyMetadata'][x]['Status']
            preacckeyvalue=keylist['AccessKeyMetadata'][x]['AccessKeyId']
            print (prevkeystatus)
            if prevkeystatus == "Inactive": 
                if preAccKey==preacckeyvalue:
                    print (preacckeyvalue)
                    iam.delete_access_key (UserName=IAM_UserName,AccessKeyId=preacckeyvalue)
                    emailmsg="PreviousKey "+preacckeyvalue+" has been deleted for user"+IAM_UserName
                    ops_sns_topic ='arn:aws:sns:us-east-1:redacted'
                    sns_send_report = boto3.client('sns',region_name='us-east-1')
                    sns_send_report.publish(TopicArn=ops_sns_topic, Message=emailmsg, Subject='Previous Key has been deleted')
                    return
                else:
                    print ("secret manager previous value doesn't match with inactive IAM key value")
            else:
                print ("previous key is still active")
        return
    except ClientError as e:
        print (e)
    #else:
    #print ("Current Key is not older than 10 days")
    
def lambda_handler(event, context):
    # TODO implement
    faction=event ["action"]
    fuser_name=event ["username"]
    if faction == "create":
        status = create_key(fuser_name)
        print (status)
    elif faction == "deactivate":
        status = deactive_key(fuser_name)
        print (status)
    elif faction == "delete":
        status = delete_key(fuser_name)
        print (status)

when testing the function I get the below error message:

Response
{
  "errorMessage": "'action'",
  "errorType": "KeyError",
  "stackTrace": [
    [
      "/var/task/lambda_function.py",
      108,
      "lambda_handler",
      "faction=event [\"action\"]"
    ]
  ]
}

Function Logs
START RequestId: 45b13b13-e992-40fe-b2e8-1f2cc53a86e5 Version: $LATEST
'action': KeyError
Traceback (most recent call last):
  File "/var/task/lambda_function.py", line 108, in lambda_handler
    faction=event ["action"]
KeyError: 'action'

I have the following policies on the role and group for my user:

IAMReadOnlyAccess

AmazonSNSFullAccess

and a custom policy with the following actions:

"iam:ListUsers",
"iam:CreateAccessKey",
"iam:DeleteAccessKey",
"iam:GetAccessKeyLastUsed",
"iam:GetUser",
"iam:ListAccessKeys",
"iam:UpdateAccessKey"
        

My EventBridge has the constant (JSON text) as {"action":"create","username":"secmanagert3"}

Looking to see why I keep getting errors on the lambda handler

Edit:

After printing out the environment variables and even, I have these function logs:

Function Logs
START RequestId: c5cabedf-d806-4ca5-a8c6-1ded84c39a39 Version: $LATEST
## ENVIRONMENT VARIABLES
environ({'PATH': '/var/lang/bin:/usr/local/bin:/usr/bin/:/bin:/opt/bin', 'LD_LIBRARY_PATH': '/var/lang/lib:/lib64:/usr/lib64:/var/runtime:/var/runtime/lib:/var/task:/var/task/lib:/opt/lib', 'LANG': 'en_US.UTF-8', 'TZ': ':UTC', '_HANDLER': 'lambda_function.lambda_handler', 'LAMBDA_TASK_ROOT': '/var/task', 'LAMBDA_RUNTIME_DIR': '/var/runtime', 'AWS_REGION': 'us-east-2', 'AWS_DEFAULT_REGION': 'us-east-2', 'AWS_LAMBDA_LOG_GROUP_NAME': '/aws/lambda/AutomatedKeyRotation', 'AWS_LAMBDA_LOG_STREAM_NAME': '2021/10/14/[$LATEST]7f05c89773e240788lda232ec5dh8hg04', 'AWS_LAMBDA_FUNCTION_NAME': 'AutomatedKeyRotation', 'AWS_LAMBDA_FUNCTION_MEMORY_SIZE': '128', 'AWS_LAMBDA_FUNCTION_VERSION': '$LATEST', '_AWS_XRAY_DAEMON_ADDRESS': 'xxx.xxx.xx.xxx', '_AWS_XRAY_DAEMON_PORT': '2000', 'AWS_XRAY_DAEMON_ADDRESS': 'xxx.xxx.xx.xxx:2000', 'AWS_XRAY_CONTEXT_MISSING': 'LOG_ERROR', '_X_AMZN_TRACE_ID': 'Root=1-61686a72-0v9fgta25cb9ca19568ae978;Parent=523645975780233;Sampled=0', 'AWS_EXECUTION_ENV': 'AWS_Lambda_python3.6', 'AWS_LAMBDA_INITIALIZATION_TYPE': 'on-demand', 'AWS_ACCESS_KEY_ID': 'key-id-number', 'AWS_SECRET_ACCESS_KEY': 'top-secret-key', 'AWS_SESSION_TOKEN': 'very-long-token', 'PYTHONPATH': '/var/runtime'})
## EVENT
{'key1': 'value1', 'key2': 'value2', 'key3': 'value3'}
'action': KeyError
Traceback (most recent call last):
  File "/var/task/lambda_function.py", line 112, in lambda_handler
    faction=event["action"]
KeyError: 'action'

Answer 1

As you can see from the log file, your event doesn't have action and username variables. That's why you're getting the KeyError.

The problem is that you are testing this by running a test from the Lambda function, and not through the Cloudwatch. To solve this:

1. In your Lambda function, open the "Test" tab. There, you can see what your event looks like. You can either manually change it, to add the values you need in the JSON, or you can choose from given templates (among others, there's Cloudwatch as a template). Once you added action and username to the JSON, it won't throw this error

2. You can create a Cloudwatch event, as instructed in the post that you shared, and invoke that event. That way, you will see exactly what the event will look like when you actually invoke it in production.