“AWS offers the data lake solution, which is an automated reference implementation that deploys a highly available, cost-effective data lake architecture on the AWS Cloud along with a user-friendly console for searching and requesting datasets.” (click here for more details).
Apart from a web-console and CLI interface, the data lake solution also has an API that you can use to interact with it. Unfortunatley there is little or no documentation of the API. You have to reverse engineer the Lambda fucntions that power the various microservices of the API. The documentation that does come with the lake gives a brief overview about how to work with the data lake API, including the following Javascript that shows how to generate an authorization key to sign your http requests to the API:
let Base64 = require('js-base64').Base64; let moment = require('moment'); let crypto = require('crypto'); function getSignatureKey(accessKey, secretKey, dateStamp, apiEndpoint) { let kDate = crypto.createHmac('sha256', "DATALAKE4" + secretKey); kDate.update(moment().utc().format("YYYYMMDD")); let kEndpoint = crypto.createHmac('sha256', kDate.digest('base64')); kEndpoint.update(apiEndpoint); let kService = crypto.createHmac('sha256', kEndpoint.digest('base64')); kService.update('datalake'); let kSigning = crypto.createHmac('sha256', kService.digest('base64')); kSigning.update("datalake4_request"); let _signature = kSigning.digest('base64'); let _apiKey = [accessKey, _signature].join(':'); let _authKey = Base64.encode(_apiKey); return ['ak', _authKey].join(':'); }
This blog post gives a python version of the above code to generate the authorization key:
def sign(key, msg): """ :param key: :param msg: :return: """ return hmac.new(key, msg.encode('utf-8'), hashlib.sha256).digest() def get_authorization_key(access_key, secret_key, date_stamp, api_end_point, service_name): """ :param access_key: :param secret_key: :param date_stamp: :param api_end_point: :param service_name: :return: """ k_date = base64.b64encode(sign(('DATALAKE4' + secret_key).encode('utf-8'), date_stamp)) k_endpoint = base64.b64encode(sign(k_date, api_end_point)) k_service = base64.b64encode(sign(k_endpoint, service_name)) k_signing = base64.b64encode(sign(k_service, 'datalake4_request')) api_key = access_key.encode('utf-8') + b':' + k_signing auth_key = "ak:{}".format(base64.b64encode(api_key).decode('utf-8')) logger.info(auth_key) return auth_key
