Skip to content

Commit db923dd

Browse files
go-to-kgo-to-k
authored
feat(glue): add ExecutionClass for FLEX (#26203)
This PR adds ExecutionClass for Glue job's L2 construct. This allows you to specify `FLEX` option. Closes #22224 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
1 parent 1f81718 commit db923dd

File tree

8 files changed

+720
-174
lines changed

8 files changed

+720
-174
lines changed

‎packages/@aws-cdk/aws-glue-alpha/README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ The `glue.JobExecutable` allows you to specify the type of job, the language to
2727

2828
`glue.Code` allows you to refer to the different code assets required by the job, either from an existing S3 location or from a local file path.
2929

30+
`glue.ExecutionClass` allows you to specify `FLEX` or `STANDARD`. `FLEX` is appropriate for non-urgent jobs such as pre-production jobs, testing, and one-time data loads.
31+
3032
### Spark Jobs
3133

3234
These jobs run in an Apache Spark environment managed by AWS Glue.

‎packages/@aws-cdk/aws-glue-alpha/lib/job.ts

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import * as logs from 'aws-cdk-lib/aws-logs';
55
import * as s3 from 'aws-cdk-lib/aws-s3';
66
import * as cdk from 'aws-cdk-lib/core';
77
import * as constructs from 'constructs';
8-
import { Code, JobExecutable, JobExecutableConfig, JobType } from '.';
8+
import { Code, GlueVersion, JobExecutable, JobExecutableConfig, JobType } from '.';
99
import { IConnection } from './connection';
1010
import { CfnJob } from 'aws-cdk-lib/aws-glue';
1111
import { ISecurityConfiguration } from './security-configuration';
@@ -129,6 +129,26 @@ export enum MetricType {
129129
COUNT = 'count',
130130
}
131131

132+
/**
133+
* The ExecutionClass whether the job is run with a standard or flexible execution class.
134+
*
135+
* @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-jobs-job.html#aws-glue-api-jobs-job-Job
136+
* @see https://docs.aws.amazon.com/glue/latest/dg/add-job.html
137+
*/
138+
export enum ExecutionClass {
139+
/**
140+
* The flexible execution class is appropriate for time-insensitive jobs whose start
141+
* and completion times may vary.
142+
*/
143+
FLEX = 'FLEX',
144+
145+
/**
146+
* The standard execution class is ideal for time-sensitive workloads that require fast job
147+
* startup and dedicated resources.
148+
*/
149+
STANDARD = 'STANDARD',
150+
}
151+
132152
/**
133153
* Interface representing a created or an imported `Job`.
134154
*/
@@ -600,6 +620,16 @@ export interface JobProps {
600620
* @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
601621
*/
602622
readonly continuousLogging?: ContinuousLoggingProps,
623+
624+
/**
625+
* The ExecutionClass whether the job is run with a standard or flexible execution class.
626+
*
627+
* @default - STANDARD
628+
*
629+
* @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-jobs-job.html#aws-glue-api-jobs-job-Job
630+
* @see https://docs.aws.amazon.com/glue/latest/dg/add-job.html
631+
*/
632+
readonly executionClass?: ExecutionClass,
603633
}
604634

605635
/**
@@ -677,6 +707,18 @@ export class Job extends JobBase {
677707
...this.checkNoReservedArgs(props.defaultArguments),
678708
};
679709

710+
if (props.executionClass === ExecutionClass.FLEX) {
711+
if (executable.type !== JobType.ETL) {
712+
throw new Error('FLEX ExecutionClass is only available for JobType.ETL jobs');
713+
}
714+
if ([GlueVersion.V0_9, GlueVersion.V1_0, GlueVersion.V2_0].includes(executable.glueVersion)) {
715+
throw new Error('FLEX ExecutionClass is only available for GlueVersion 3.0 or later');
716+
}
717+
if (props.workerType && (props.workerType !== WorkerType.G_1X && props.workerType !== WorkerType.G_2X)) {
718+
throw new Error('FLEX ExecutionClass is only available for WorkerType G_1X or G_2X');
719+
}
720+
}
721+
680722
const jobResource = new CfnJob(this, 'Resource', {
681723
name: props.jobName,
682724
description: props.description,
@@ -692,6 +734,7 @@ export class Job extends JobBase {
692734
numberOfWorkers: props.workerCount,
693735
maxCapacity: props.maxCapacity,
694736
maxRetries: props.maxRetries,
737+
executionClass: props.executionClass,
695738
executionProperty: props.maxConcurrentRuns ? { maxConcurrentRuns: props.maxConcurrentRuns } : undefined,
696739
notificationProperty: props.notifyDelayAfter ? { notifyDelayAfter: props.notifyDelayAfter.toMinutes() } : undefined,
697740
timeout: props.timeout?.toMinutes(),

‎packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.assets.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,15 @@
1414
}
1515
}
1616
},
17-
"e99fb38377ba41ea9e74da162cf01b6821baa17e8e3d003c711b03d822356b89": {
17+
"4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a": {
1818
"source": {
1919
"path": "aws-glue-job.template.json",
2020
"packaging": "file"
2121
},
2222
"destinations": {
2323
"current_account-current_region": {
2424
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
25-
"objectKey": "e99fb38377ba41ea9e74da162cf01b6821baa17e8e3d003c711b03d822356b89.json",
25+
"objectKey": "4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a.json",
2626
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
2727
}
2828
}

‎packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/aws-glue-job.template.json

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@
176176
"arg2": "value2",
177177
"--conf": "valueConf"
178178
},
179+
"ExecutionClass": "STANDARD",
179180
"ExecutionProperty": {
180181
"MaxConcurrentRuns": 2
181182
},
@@ -533,6 +534,7 @@
533534
"arg2": "value2",
534535
"--conf": "valueConf"
535536
},
537+
"ExecutionClass": "STANDARD",
536538
"ExecutionProperty": {
537539
"MaxConcurrentRuns": 2
538540
},
@@ -890,6 +892,7 @@
890892
"arg2": "value2",
891893
"--conf": "valueConf"
892894
},
895+
"ExecutionClass": "STANDARD",
893896
"ExecutionProperty": {
894897
"MaxConcurrentRuns": 2
895898
},
@@ -1448,6 +1451,129 @@
14481451
},
14491452
"WorkerType": "Z.2X"
14501453
}
1454+
},
1455+
"EtlJobWithFLEXServiceRoleBA7C99A5": {
1456+
"Type": "AWS::IAM::Role",
1457+
"Properties": {
1458+
"AssumeRolePolicyDocument": {
1459+
"Statement": [
1460+
{
1461+
"Action": "sts:AssumeRole",
1462+
"Effect": "Allow",
1463+
"Principal": {
1464+
"Service": "glue.amazonaws.com"
1465+
}
1466+
}
1467+
],
1468+
"Version": "2012-10-17"
1469+
},
1470+
"ManagedPolicyArns": [
1471+
{
1472+
"Fn::Join": [
1473+
"",
1474+
[
1475+
"arn:",
1476+
{
1477+
"Ref": "AWS::Partition"
1478+
},
1479+
":iam::aws:policy/service-role/AWSGlueServiceRole"
1480+
]
1481+
]
1482+
}
1483+
]
1484+
}
1485+
},
1486+
"EtlJobWithFLEXServiceRoleDefaultPolicyEA63C5FE": {
1487+
"Type": "AWS::IAM::Policy",
1488+
"Properties": {
1489+
"PolicyDocument": {
1490+
"Statement": [
1491+
{
1492+
"Action": [
1493+
"s3:GetBucket*",
1494+
"s3:GetObject*",
1495+
"s3:List*"
1496+
],
1497+
"Effect": "Allow",
1498+
"Resource": [
1499+
{
1500+
"Fn::Join": [
1501+
"",
1502+
[
1503+
"arn:",
1504+
{
1505+
"Ref": "AWS::Partition"
1506+
},
1507+
":s3:::",
1508+
{
1509+
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
1510+
},
1511+
"/*"
1512+
]
1513+
]
1514+
},
1515+
{
1516+
"Fn::Join": [
1517+
"",
1518+
[
1519+
"arn:",
1520+
{
1521+
"Ref": "AWS::Partition"
1522+
},
1523+
":s3:::",
1524+
{
1525+
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
1526+
}
1527+
]
1528+
]
1529+
}
1530+
]
1531+
}
1532+
],
1533+
"Version": "2012-10-17"
1534+
},
1535+
"PolicyName": "EtlJobWithFLEXServiceRoleDefaultPolicyEA63C5FE",
1536+
"Roles": [
1537+
{
1538+
"Ref": "EtlJobWithFLEXServiceRoleBA7C99A5"
1539+
}
1540+
]
1541+
}
1542+
},
1543+
"EtlJobWithFLEX928B4212": {
1544+
"Type": "AWS::Glue::Job",
1545+
"Properties": {
1546+
"Command": {
1547+
"Name": "glueetl",
1548+
"PythonVersion": "3",
1549+
"ScriptLocation": {
1550+
"Fn::Join": [
1551+
"",
1552+
[
1553+
"s3://",
1554+
{
1555+
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
1556+
},
1557+
"/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py"
1558+
]
1559+
]
1560+
}
1561+
},
1562+
"Role": {
1563+
"Fn::GetAtt": [
1564+
"EtlJobWithFLEXServiceRoleBA7C99A5",
1565+
"Arn"
1566+
]
1567+
},
1568+
"DefaultArguments": {
1569+
"--job-language": "python"
1570+
},
1571+
"ExecutionClass": "FLEX",
1572+
"GlueVersion": "3.0",
1573+
"Name": "EtlJobWithFLEX",
1574+
"NumberOfWorkers": 10,
1575+
"WorkerType": "G.1X"
1576+
}
14511577
}
14521578
},
14531579
"Parameters": {

‎packages/@aws-cdk/aws-glue-alpha/test/integ.job.js.snapshot/manifest.json

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
"validateOnSynth": false,
1818
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}",
1919
"cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}",
20-
"stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/e99fb38377ba41ea9e74da162cf01b6821baa17e8e3d003c711b03d822356b89.json",
20+
"stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/4ff2e5500ce87081dfbf121cb29a8c64ffc6edc276257bb420e68abbb49af40a.json",
2121
"requiresBootstrapStackVersion": 6,
2222
"bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version",
2323
"additionalDependencies": [
@@ -231,6 +231,24 @@
231231
"data": "RayJob2F7864D9"
232232
}
233233
],
234+
"/aws-glue-job/EtlJobWithFLEX/ServiceRole/Resource": [
235+
{
236+
"type": "aws:cdk:logicalId",
237+
"data": "EtlJobWithFLEXServiceRoleBA7C99A5"
238+
}
239+
],
240+
"/aws-glue-job/EtlJobWithFLEX/ServiceRole/DefaultPolicy/Resource": [
241+
{
242+
"type": "aws:cdk:logicalId",
243+
"data": "EtlJobWithFLEXServiceRoleDefaultPolicyEA63C5FE"
244+
}
245+
],
246+
"/aws-glue-job/EtlJobWithFLEX/Resource": [
247+
{
248+
"type": "aws:cdk:logicalId",
249+
"data": "EtlJobWithFLEX928B4212"
250+
}
251+
],
234252
"/aws-glue-job/BootstrapVersion": [
235253
{
236254
"type": "aws:cdk:logicalId",

0 commit comments

Comments
 (0)