Class: AWSCDK::Glue::CfnJobProps

Inherits:
Jsii::Struct
  • Object
show all
Defined in:
glue/cfn_job_props.rb

Overview

Properties for defining a CfnJob.

See Also:

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(command:, role:, allocated_capacity: nil, connections: nil, default_arguments: nil, description: nil, execution_class: nil, execution_property: nil, glue_version: nil, job_mode: nil, job_run_queuing_enabled: nil, log_uri: nil, maintenance_window: nil, max_capacity: nil, max_retries: nil, name: nil, non_overridable_arguments: nil, notification_property: nil, number_of_workers: nil, security_configuration: nil, tags: nil, timeout: nil, worker_type: nil) ⇒ CfnJobProps

Returns a new instance of CfnJobProps.

Parameters:

  • command (AWSCDK::IResolvable, AWSCDK::Glue::CfnJob::JobCommandProperty) —

    The code that executes a job.

  • role (String, AWSCDK::Interfaces::AWSIAM::IRoleRef) —

    The name or Amazon Resource Name (ARN) of the IAM role associated with this job.

  • allocated_capacity (Numeric, nil) (defaults to: nil) —

    This parameter is no longer supported. Use MaxCapacity instead.

  • connections (AWSCDK::IResolvable, AWSCDK::Glue::CfnJob::ConnectionsListProperty, nil) (defaults to: nil) —

    The connections used for this job.

  • default_arguments (Object, nil) (defaults to: nil) —

    The default arguments for this job, specified as name-value pairs.

  • description (String, nil) (defaults to: nil) —

    A description of the job.

  • execution_class (String, nil) (defaults to: nil) —

    Indicates whether the job is run with a standard or flexible execution class.

  • execution_property (AWSCDK::IResolvable, AWSCDK::Glue::CfnJob::ExecutionPropertyProperty, nil) (defaults to: nil) —

    The maximum number of concurrent runs that are allowed for this job.

  • glue_version (String, nil) (defaults to: nil) —

    Glue version determines the versions of Apache Spark and Python that AWS Glue supports.

  • job_mode (String, nil) (defaults to: nil) —

    A mode that describes how a job was created. Valid values are:.

  • job_run_queuing_enabled (Boolean, AWSCDK::IResolvable, nil) (defaults to: nil) —

    Specifies whether job run queuing is enabled for the job runs for this job.

  • log_uri (String, nil) (defaults to: nil) —

    This field is reserved for future use.

  • maintenance_window (String, nil) (defaults to: nil) —

    This field specifies a day of the week and hour for a maintenance window for streaming jobs.

  • max_capacity (Numeric, nil) (defaults to: nil) —

    The number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.

  • max_retries (Numeric, nil) (defaults to: nil) —

    The maximum number of times to retry this job after a JobRun fails.

  • name (String, nil) (defaults to: nil) —

    The name you assign to this job definition.

  • non_overridable_arguments (Object, nil) (defaults to: nil) —

    Non-overridable arguments for this job, specified as name-value pairs.

  • notification_property (AWSCDK::IResolvable, AWSCDK::Glue::CfnJob::NotificationPropertyProperty, nil) (defaults to: nil) —

    Specifies configuration properties of a notification.

  • number_of_workers (Numeric, nil) (defaults to: nil) —

    The number of workers of a defined workerType that are allocated when a job runs.

  • security_configuration (String, nil) (defaults to: nil) —

    The name of the SecurityConfiguration structure to be used with this job.

  • tags (Object, nil) (defaults to: nil) —

    The tags to use with this job.

  • timeout (Numeric, nil) (defaults to: nil) —

    The job timeout in minutes.

  • worker_type (String, nil) (defaults to: nil) —

    The type of predefined worker that is allocated when a job runs.



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'glue/cfn_job_props.rb', line 32

def initialize(command:, role:, allocated_capacity: nil, connections: nil, default_arguments: nil, description: nil, execution_class: nil, execution_property: nil, glue_version: nil, job_mode: nil, job_run_queuing_enabled: nil, log_uri: nil, maintenance_window: nil, max_capacity: nil, max_retries: nil, name: nil, non_overridable_arguments: nil, notification_property: nil, number_of_workers: nil, security_configuration: nil, tags: nil, timeout: nil, worker_type: nil)
  @command = command.is_a?(Hash) ? ::AWSCDK::Glue::CfnJob::JobCommandProperty.new(**command.transform_keys(&:to_sym)) : command
  Jsii::Type.check_type(@command, JSON.parse(Base64.strict_decode64("eyJ1bmlvbiI6eyJ0eXBlcyI6W3siZnFuIjoiYXdzLWNkay1saWIuSVJlc29sdmFibGUifSx7ImZxbiI6ImF3cy1jZGstbGliLmF3c19nbHVlLkNmbkpvYi5Kb2JDb21tYW5kUHJvcGVydHkifV19fQ==")), "command")
  @role = role
  Jsii::Type.check_type(@role, JSON.parse(Base64.strict_decode64("eyJ1bmlvbiI6eyJ0eXBlcyI6W3sicHJpbWl0aXZlIjoic3RyaW5nIn0seyJmcW4iOiJhd3MtY2RrLWxpYi5pbnRlcmZhY2VzLmF3c19pYW0uSVJvbGVSZWYifV19fQ==")), "role")
  @allocated_capacity = allocated_capacity
  Jsii::Type.check_type(@allocated_capacity, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJudW1iZXIifQ==")), "allocatedCapacity") unless @allocated_capacity.nil?
  @connections = connections.is_a?(Hash) ? ::AWSCDK::Glue::CfnJob::ConnectionsListProperty.new(**connections.transform_keys(&:to_sym)) : connections
  Jsii::Type.check_type(@connections, JSON.parse(Base64.strict_decode64("eyJ1bmlvbiI6eyJ0eXBlcyI6W3siZnFuIjoiYXdzLWNkay1saWIuSVJlc29sdmFibGUifSx7ImZxbiI6ImF3cy1jZGstbGliLmF3c19nbHVlLkNmbkpvYi5Db25uZWN0aW9uc0xpc3RQcm9wZXJ0eSJ9XX19")), "connections") unless @connections.nil?
  @default_arguments = default_arguments
  Jsii::Type.check_type(@default_arguments, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJhbnkifQ==")), "defaultArguments") unless @default_arguments.nil?
  @description = description
  Jsii::Type.check_type(@description, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJzdHJpbmcifQ==")), "description") unless @description.nil?
  @execution_class = execution_class
  Jsii::Type.check_type(@execution_class, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJzdHJpbmcifQ==")), "executionClass") unless @execution_class.nil?
  @execution_property = execution_property.is_a?(Hash) ? ::AWSCDK::Glue::CfnJob::ExecutionPropertyProperty.new(**execution_property.transform_keys(&:to_sym)) : execution_property
  Jsii::Type.check_type(@execution_property, JSON.parse(Base64.strict_decode64("eyJ1bmlvbiI6eyJ0eXBlcyI6W3siZnFuIjoiYXdzLWNkay1saWIuSVJlc29sdmFibGUifSx7ImZxbiI6ImF3cy1jZGstbGliLmF3c19nbHVlLkNmbkpvYi5FeGVjdXRpb25Qcm9wZXJ0eVByb3BlcnR5In1dfX0=")), "executionProperty") unless @execution_property.nil?
  @glue_version = glue_version
  Jsii::Type.check_type(@glue_version, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJzdHJpbmcifQ==")), "glueVersion") unless @glue_version.nil?
  @job_mode = job_mode
  Jsii::Type.check_type(@job_mode, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJzdHJpbmcifQ==")), "jobMode") unless @job_mode.nil?
  @job_run_queuing_enabled = job_run_queuing_enabled
  Jsii::Type.check_type(@job_run_queuing_enabled, JSON.parse(Base64.strict_decode64("eyJ1bmlvbiI6eyJ0eXBlcyI6W3sicHJpbWl0aXZlIjoiYm9vbGVhbiJ9LHsiZnFuIjoiYXdzLWNkay1saWIuSVJlc29sdmFibGUifV19fQ==")), "jobRunQueuingEnabled") unless @job_run_queuing_enabled.nil?
  @log_uri = log_uri
  Jsii::Type.check_type(@log_uri, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJzdHJpbmcifQ==")), "logUri") unless @log_uri.nil?
  @maintenance_window = maintenance_window
  Jsii::Type.check_type(@maintenance_window, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJzdHJpbmcifQ==")), "maintenanceWindow") unless @maintenance_window.nil?
  @max_capacity = max_capacity
  Jsii::Type.check_type(@max_capacity, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJudW1iZXIifQ==")), "maxCapacity") unless @max_capacity.nil?
  @max_retries = max_retries
  Jsii::Type.check_type(@max_retries, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJudW1iZXIifQ==")), "maxRetries") unless @max_retries.nil?
  @name = name
  Jsii::Type.check_type(@name, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJzdHJpbmcifQ==")), "name") unless @name.nil?
  @non_overridable_arguments = non_overridable_arguments
  Jsii::Type.check_type(@non_overridable_arguments, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJhbnkifQ==")), "nonOverridableArguments") unless @non_overridable_arguments.nil?
  @notification_property = notification_property.is_a?(Hash) ? ::AWSCDK::Glue::CfnJob::NotificationPropertyProperty.new(**notification_property.transform_keys(&:to_sym)) : notification_property
  Jsii::Type.check_type(@notification_property, JSON.parse(Base64.strict_decode64("eyJ1bmlvbiI6eyJ0eXBlcyI6W3siZnFuIjoiYXdzLWNkay1saWIuSVJlc29sdmFibGUifSx7ImZxbiI6ImF3cy1jZGstbGliLmF3c19nbHVlLkNmbkpvYi5Ob3RpZmljYXRpb25Qcm9wZXJ0eVByb3BlcnR5In1dfX0=")), "notificationProperty") unless @notification_property.nil?
  @number_of_workers = number_of_workers
  Jsii::Type.check_type(@number_of_workers, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJudW1iZXIifQ==")), "numberOfWorkers") unless @number_of_workers.nil?
  @security_configuration = security_configuration
  Jsii::Type.check_type(@security_configuration, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJzdHJpbmcifQ==")), "securityConfiguration") unless @security_configuration.nil?
  @tags = tags
  Jsii::Type.check_type(@tags, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJhbnkifQ==")), "tags") unless @tags.nil?
  @timeout = timeout
  Jsii::Type.check_type(@timeout, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJudW1iZXIifQ==")), "timeout") unless @timeout.nil?
  @worker_type = worker_type
  Jsii::Type.check_type(@worker_type, JSON.parse(Base64.strict_decode64("eyJwcmltaXRpdmUiOiJzdHJpbmcifQ==")), "workerType") unless @worker_type.nil?
end

Instance Attribute Details

#allocated_capacity ⇒ Numeric? (readonly)

This parameter is no longer supported. Use MaxCapacity instead.

The number of capacity units that are allocated to this job.

Returns:

  • (Numeric, nil)

See Also:



97
98
99
 
# File 'glue/cfn_job_props.rb', line 97

def allocated_capacity
  @allocated_capacity
end

#command ⇒ AWSCDK::IResolvable, AWSCDK::Glue::CfnJob::JobCommandProperty (readonly)

The code that executes a job.

Returns:

See Also:



85
86
87
 
# File 'glue/cfn_job_props.rb', line 85

def command
  @command
end

#connections ⇒ AWSCDK::IResolvable, ... (readonly)

The connections used for this job.

Returns:

See Also:



102
103
104
 
# File 'glue/cfn_job_props.rb', line 102

def connections
  @connections
end

#default_arguments ⇒ Object? (readonly)

The default arguments for this job, specified as name-value pairs.

You can specify arguments here that your own job-execution script consumes, in addition to arguments that AWS Glue itself consumes.

For information about how to specify and consume your own job arguments, see Calling AWS Glue APIs in Python in the AWS Glue Developer Guide .

For information about the key-value pairs that AWS Glue consumes to set up your job, see Special Parameters Used by AWS Glue in the AWS Glue Developer Guide .

Returns:

  • (Object, nil)

See Also:



113
114
115
 
# File 'glue/cfn_job_props.rb', line 113

def default_arguments
  @default_arguments
end

#description ⇒ String? (readonly)

A description of the job.

Returns:

  • (String, nil)

See Also:



118
119
120
 
# File 'glue/cfn_job_props.rb', line 118

def description
  @description
end

#execution_class ⇒ String? (readonly)

Indicates whether the job is run with a standard or flexible execution class.

The standard execution class is ideal for time-sensitive workloads that require fast job startup and dedicated resources.

The flexible execution class is appropriate for time-insensitive jobs whose start and completion times may vary.

Only jobs with AWS Glue version 3.0 and above and command type glueetl will be allowed to set ExecutionClass to FLEX . The flexible execution class is available for Spark jobs.

Returns:

  • (String, nil)

See Also:



129
130
131
 
# File 'glue/cfn_job_props.rb', line 129

def execution_class
  @execution_class
end

#execution_property ⇒ AWSCDK::IResolvable, ... (readonly)

The maximum number of concurrent runs that are allowed for this job.

Returns:

See Also:



134
135
136
 
# File 'glue/cfn_job_props.rb', line 134

def execution_property
  @execution_property
end

#glue_version ⇒ String? (readonly)

Glue version determines the versions of Apache Spark and Python that AWS Glue supports.

The Python version indicates the version supported for jobs of type Spark.

For more information about the available AWS Glue versions and corresponding Spark and Python versions, see Glue version in the developer guide.

Jobs that are created without specifying a Glue version default to the latest Glue version available.

Returns:

  • (String, nil)

See Also:



145
146
147
 
# File 'glue/cfn_job_props.rb', line 145

def glue_version
  @glue_version
end

#job_mode ⇒ String? (readonly)

A mode that describes how a job was created. Valid values are:.

  • SCRIPT - The job was created using the AWS Glue Studio script editor.
  • VISUAL - The job was created using the AWS Glue Studio visual editor.
  • NOTEBOOK - The job was created using an interactive sessions notebook.

When the JobMode field is missing or null, SCRIPT is assigned as the default value.

Returns:

  • (String, nil)

See Also:



156
157
158
 
# File 'glue/cfn_job_props.rb', line 156

def job_mode
  @job_mode
end

#job_run_queuing_enabled ⇒ Boolean, ... (readonly)

Specifies whether job run queuing is enabled for the job runs for this job.

A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing.

If this field does not match the value set in the job run, then the value from the job run field will be used.

Returns:

See Also:



165
166
167
 
# File 'glue/cfn_job_props.rb', line 165

def job_run_queuing_enabled
  @job_run_queuing_enabled
end

#log_uri ⇒ String? (readonly)

This field is reserved for future use.

Returns:

  • (String, nil)

See Also:



170
171
172
 
# File 'glue/cfn_job_props.rb', line 170

def log_uri
  @log_uri
end

#maintenance_window ⇒ String? (readonly)

This field specifies a day of the week and hour for a maintenance window for streaming jobs.

AWS Glue periodically performs maintenance activities. During these maintenance windows, AWS Glue will need to restart your streaming jobs.

AWS Glue will restart the job within 3 hours of the specified maintenance window. For instance, if you set up the maintenance window for Monday at 10:00AM GMT, your jobs will be restarted between 10:00AM GMT to 1:00PM GMT.

Returns:

  • (String, nil)

See Also:



179
180
181
 
# File 'glue/cfn_job_props.rb', line 179

def maintenance_window
  @maintenance_window
end

#max_capacity ⇒ Numeric? (readonly)

The number of AWS Glue data processing units (DPUs) that can be allocated when this job runs.

A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory.

Do not set Max Capacity if using WorkerType and NumberOfWorkers .

The value that can be allocated for MaxCapacity depends on whether you are running a Python shell job or an Apache Spark ETL job:

  • When you specify a Python shell job ( JobCommand.Name ="pythonshell"), you can allocate either 0.0625 or 1 DPU. The default is 0.0625 DPU.
  • When you specify an Apache Spark ETL job ( JobCommand.Name ="glueetl"), you can allocate from 2 to 100 DPUs. The default is 10 DPUs. This job type cannot have a fractional DPU allocation.

Returns:

  • (Numeric, nil)

See Also:



193
194
195
 
# File 'glue/cfn_job_props.rb', line 193

def max_capacity
  @max_capacity
end

#max_retries ⇒ Numeric? (readonly)

The maximum number of times to retry this job after a JobRun fails.

Returns:

  • (Numeric, nil)

See Also:



198
199
200
 
# File 'glue/cfn_job_props.rb', line 198

def max_retries
  @max_retries
end

#name ⇒ String? (readonly)

The name you assign to this job definition.

Returns:

  • (String, nil)

See Also:



203
204
205
 
# File 'glue/cfn_job_props.rb', line 203

def name
  @name
end

#non_overridable_arguments ⇒ Object? (readonly)

Non-overridable arguments for this job, specified as name-value pairs.

Returns:

  • (Object, nil)

See Also:



208
209
210
 
# File 'glue/cfn_job_props.rb', line 208

def non_overridable_arguments
  @non_overridable_arguments
end

#notification_property ⇒ AWSCDK::IResolvable, ... (readonly)

Specifies configuration properties of a notification.

Returns:

See Also:



213
214
215
 
# File 'glue/cfn_job_props.rb', line 213

def notification_property
  @notification_property
end

#number_of_workers ⇒ Numeric? (readonly)

The number of workers of a defined workerType that are allocated when a job runs.

The maximum number of workers you can define are 299 for G.1X , and 149 for G.2X .

Returns:

  • (Numeric, nil)

See Also:



220
221
222
 
# File 'glue/cfn_job_props.rb', line 220

def number_of_workers
  @number_of_workers
end

#role ⇒ String, AWSCDK::Interfaces::AWSIAM::IRoleRef (readonly)

The name or Amazon Resource Name (ARN) of the IAM role associated with this job.

Returns:

See Also:



90
91
92
 
# File 'glue/cfn_job_props.rb', line 90

def role
  @role
end

#security_configuration ⇒ String? (readonly)

The name of the SecurityConfiguration structure to be used with this job.

Returns:

  • (String, nil)

See Also:



225
226
227
 
# File 'glue/cfn_job_props.rb', line 225

def security_configuration
  @security_configuration
end

#tags ⇒ Object? (readonly)

The tags to use with this job.

Returns:

  • (Object, nil)

See Also:



230
231
232
 
# File 'glue/cfn_job_props.rb', line 230

def tags
  @tags
end

#timeout ⇒ Numeric? (readonly)

The job timeout in minutes.

This is the maximum time that a job run can consume resources before it is terminated and enters TIMEOUT status. The default is 2,880 minutes (48 hours).

Returns:

  • (Numeric, nil)

See Also:



237
238
239
 
# File 'glue/cfn_job_props.rb', line 237

def timeout
  @timeout
end

#worker_type ⇒ String? (readonly)

The type of predefined worker that is allocated when a job runs.

AWS Glue provides multiple worker types to accommodate different workload requirements:

G Worker Types (General-purpose compute workers):

  • G.1X: 1 DPU (4 vCPUs, 16 GB memory, 94GB disk)
  • G.2X: 2 DPU (8 vCPUs, 32 GB memory, 138GB disk)
  • G.4X: 4 DPU (16 vCPUs, 64 GB memory, 256GB disk)
  • G.8X: 8 DPU (32 vCPUs, 128 GB memory, 512GB disk)
  • G.12X: 12 DPU (48 vCPUs, 192 GB memory, 768GB disk)
  • G.16X: 16 DPU (64 vCPUs, 256 GB memory, 1024GB disk)

R Worker Types (Memory-optimized workers):

  • R.1X: 1 M-DPU (4 vCPUs, 32 GB memory)
  • R.2X: 2 M-DPU (8 vCPUs, 64 GB memory)
  • R.4X: 4 M-DPU (16 vCPUs, 128 GB memory)
  • R.8X: 8 M-DPU (32 vCPUs, 256 GB memory)

Returns:

  • (String, nil)

See Also:



260
261
262
 
# File 'glue/cfn_job_props.rb', line 260

def worker_type
  @worker_type
end

Class Method Details

.jsii_properties ⇒ Object 



262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
 
# File 'glue/cfn_job_props.rb', line 262

def self.jsii_properties
  {
    :command => "command",
    :role => "role",
    :allocated_capacity => "allocatedCapacity",
    :connections => "connections",
    :default_arguments => "defaultArguments",
    :description => "description",
    :execution_class => "executionClass",
    :execution_property => "executionProperty",
    :glue_version => "glueVersion",
    :job_mode => "jobMode",
    :job_run_queuing_enabled => "jobRunQueuingEnabled",
    :log_uri => "logUri",
    :maintenance_window => "maintenanceWindow",
    :max_capacity => "maxCapacity",
    :max_retries => "maxRetries",
    :name => "name",
    :non_overridable_arguments => "nonOverridableArguments",
    :notification_property => "notificationProperty",
    :number_of_workers => "numberOfWorkers",
    :security_configuration => "securityConfiguration",
    :tags => "tags",
    :timeout => "timeout",
    :worker_type => "workerType",
  }
end

Instance Method Details

#to_jsii ⇒ Object 



290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
 
# File 'glue/cfn_job_props.rb', line 290

def to_jsii
  result = {}
  result.merge!({
    "command" => @command,
    "role" => @role,
    "allocatedCapacity" => @allocated_capacity,
    "connections" => @connections,
    "defaultArguments" => @default_arguments,
    "description" => @description,
    "executionClass" => @execution_class,
    "executionProperty" => @execution_property,
    "glueVersion" => @glue_version,
    "jobMode" => @job_mode,
    "jobRunQueuingEnabled" => @job_run_queuing_enabled,
    "logUri" => @log_uri,
    "maintenanceWindow" => @maintenance_window,
    "maxCapacity" => @max_capacity,
    "maxRetries" => @max_retries,
    "name" => @name,
    "nonOverridableArguments" => @non_overridable_arguments,
    "notificationProperty" => @notification_property,
    "numberOfWorkers" => @number_of_workers,
    "securityConfiguration" => @security_configuration,
    "tags" => @tags,
    "timeout" => @timeout,
    "workerType" => @worker_type,
  })
  result.compact
end