From c373474d2ab7bc9c0dc1e1d0de8f72bd174e6191 Mon Sep 17 00:00:00 2001 From: Inbar Shani Date: Mon, 3 Nov 2025 18:48:24 +0200 Subject: [PATCH 1/2] Add an argument to _get_workgroup_config and all functions using _get_workgroup_config to determine if the workgroup config should be retreived from the server --- awswrangler/athena/_executions.py | 10 ++++++- awswrangler/athena/_executions.pyi | 3 +++ awswrangler/athena/_read.py | 32 +++++++++++++++++++++-- awswrangler/athena/_read.pyi | 11 ++++++++ awswrangler/athena/_utils.py | 39 +++++++++++++++++++++++++--- awswrangler/athena/_write_iceberg.py | 33 ++++++++++++++++++++--- 6 files changed, 118 insertions(+), 10 deletions(-) diff --git a/awswrangler/athena/_executions.py b/awswrangler/athena/_executions.py index f3297771e..952a87128 100644 --- a/awswrangler/athena/_executions.py +++ b/awswrangler/athena/_executions.py @@ -47,6 +47,7 @@ def start_query_execution( athena_query_wait_polling_delay: float = _QUERY_WAIT_POLLING_DELAY, data_source: str | None = None, wait: bool = False, + retreive_workgroup_config: bool = True, ) -> str | dict[str, Any]: """Start a SQL Query against AWS Athena. @@ -114,6 +115,11 @@ def start_query_execution( Data Source / Catalog name. If None, 'AwsDataCatalog' will be used by default. wait Indicates whether to wait for the query to finish and return a dictionary with the query execution response. + retreive_workgroup_config + Indicates whether to use the workgroup configuration for the query execution. + If True, the workgroup configuration will be retreived and used to determine the s3 output location, encryption, and kms key. + If False, the s3 output location, encryption, and kms key will not be set and will be determined by the AWS Athena service. + Default is True. Returns ------- @@ -149,7 +155,9 @@ def start_query_execution( query_execution_id = cache_info.query_execution_id _logger.debug("Valid cache found. Retrieving...") else: - wg_config: _WorkGroupConfig = _get_workgroup_config(session=boto3_session, workgroup=workgroup) + wg_config: _WorkGroupConfig = _get_workgroup_config( + session=boto3_session, workgroup=workgroup, retreive_workgroup_config=retreive_workgroup_config + ) query_execution_id = _start_query_execution( sql=sql, wg_config=wg_config, diff --git a/awswrangler/athena/_executions.pyi b/awswrangler/athena/_executions.pyi index 073e63c8d..42a354c6a 100644 --- a/awswrangler/athena/_executions.pyi +++ b/awswrangler/athena/_executions.pyi @@ -24,6 +24,7 @@ def start_query_execution( athena_query_wait_polling_delay: float = ..., data_source: str | None = ..., wait: Literal[False] = ..., + retreive_workgroup_config: bool = ..., ) -> str: ... @overload def start_query_execution( @@ -42,6 +43,7 @@ def start_query_execution( athena_query_wait_polling_delay: float = ..., data_source: str | None = ..., wait: Literal[True], + retreive_workgroup_config: bool = ..., ) -> dict[str, Any]: ... @overload def start_query_execution( @@ -60,6 +62,7 @@ def start_query_execution( athena_query_wait_polling_delay: float = ..., data_source: str | None = ..., wait: bool, + retreive_workgroup_config: bool = ..., ) -> str | dict[str, Any]: ... def stop_query_execution(query_execution_id: str, boto3_session: boto3.Session | None = ...) -> None: ... def wait_query( diff --git a/awswrangler/athena/_read.py b/awswrangler/athena/_read.py index 766a88c99..dbdecfa10 100644 --- a/awswrangler/athena/_read.py +++ b/awswrangler/athena/_read.py @@ -321,6 +321,7 @@ def _resolve_query_without_cache_ctas( pyarrow_additional_kwargs: dict[str, Any] | None = None, execution_params: list[str] | None = None, dtype_backend: Literal["numpy_nullable", "pyarrow"] = "numpy_nullable", + retreive_workgroup_config: bool = True, ) -> pd.DataFrame | Iterator[pd.DataFrame]: ctas_query_info: dict[str, str | _QueryMetadata] = create_ctas_table( sql=sql, @@ -339,6 +340,7 @@ def _resolve_query_without_cache_ctas( boto3_session=boto3_session, params=execution_params, paramstyle="qmark", + retreive_workgroup_config=retreive_workgroup_config, ) fully_qualified_name: str = f'"{ctas_query_info["ctas_database"]}"."{ctas_query_info["ctas_table"]}"' ctas_query_metadata = cast(_QueryMetadata, ctas_query_info["ctas_query_metadata"]) @@ -379,6 +381,7 @@ def _resolve_query_without_cache_unload( pyarrow_additional_kwargs: dict[str, Any] | None = None, execution_params: list[str] | None = None, dtype_backend: Literal["numpy_nullable", "pyarrow"] = "numpy_nullable", + retreive_workgroup_config: bool = True, ) -> pd.DataFrame | Iterator[pd.DataFrame]: query_metadata = _unload( sql=sql, @@ -395,6 +398,7 @@ def _resolve_query_without_cache_unload( data_source=data_source, athena_query_wait_polling_delay=athena_query_wait_polling_delay, execution_params=execution_params, + retreive_workgroup_config=retreive_workgroup_config, ) if file_format == "PARQUET": return _fetch_parquet_result( @@ -430,8 +434,11 @@ def _resolve_query_without_cache_regular( result_reuse_configuration: dict[str, Any] | None = None, dtype_backend: Literal["numpy_nullable", "pyarrow"] = "numpy_nullable", client_request_token: str | None = None, + retreive_workgroup_config: bool = True, ) -> pd.DataFrame | Iterator[pd.DataFrame]: - wg_config: _WorkGroupConfig = _get_workgroup_config(session=boto3_session, workgroup=workgroup) + wg_config: _WorkGroupConfig = _get_workgroup_config( + session=boto3_session, workgroup=workgroup, retreive_workgroup_config=retreive_workgroup_config + ) s3_output = _get_s3_output(s3_output=s3_output, wg_config=wg_config, boto3_session=boto3_session) s3_output = s3_output[:-1] if s3_output[-1] == "/" else s3_output _logger.debug("Executing sql: %s", sql) @@ -496,6 +503,7 @@ def _resolve_query_without_cache( # noqa: PLR0913 result_reuse_configuration: dict[str, Any] | None = None, dtype_backend: Literal["numpy_nullable", "pyarrow"] = "numpy_nullable", client_request_token: str | None = None, + retreive_workgroup_config: bool = True, ) -> pd.DataFrame | Iterator[pd.DataFrame]: """ Execute a query in Athena and returns results as DataFrame, back to `read_sql_query`. @@ -530,6 +538,7 @@ def _resolve_query_without_cache( # noqa: PLR0913 pyarrow_additional_kwargs=pyarrow_additional_kwargs, execution_params=execution_params, dtype_backend=dtype_backend, + retreive_workgroup_config=retreive_workgroup_config, ) finally: catalog.delete_table_if_exists(database=ctas_database or database, table=name, boto3_session=boto3_session) @@ -558,6 +567,7 @@ def _resolve_query_without_cache( # noqa: PLR0913 pyarrow_additional_kwargs=pyarrow_additional_kwargs, execution_params=execution_params, dtype_backend=dtype_backend, + retreive_workgroup_config=retreive_workgroup_config, ) return _resolve_query_without_cache_regular( sql=sql, @@ -578,6 +588,7 @@ def _resolve_query_without_cache( # noqa: PLR0913 result_reuse_configuration=result_reuse_configuration, dtype_backend=dtype_backend, client_request_token=client_request_token, + retreive_workgroup_config=retreive_workgroup_config, ) @@ -596,8 +607,11 @@ def _unload( data_source: str | None, athena_query_wait_polling_delay: float, execution_params: list[str] | None, + retreive_workgroup_config: bool = True, ) -> _QueryMetadata: - wg_config: _WorkGroupConfig = _get_workgroup_config(session=boto3_session, workgroup=workgroup) + wg_config: _WorkGroupConfig = _get_workgroup_config( + session=boto3_session, workgroup=workgroup, retreive_workgroup_config=retreive_workgroup_config + ) s3_output: str = _get_s3_output(s3_output=path, wg_config=wg_config, boto3_session=boto3_session) s3_output = s3_output[:-1] if s3_output[-1] == "/" else s3_output # Athena does not enforce a Query Result Location for UNLOAD. Thus, the workgroup output location @@ -793,6 +807,7 @@ def read_sql_query( dtype_backend: Literal["numpy_nullable", "pyarrow"] = "numpy_nullable", s3_additional_kwargs: dict[str, Any] | None = None, pyarrow_additional_kwargs: dict[str, Any] | None = None, + retreive_workgroup_config: bool = True, ) -> pd.DataFrame | Iterator[pd.DataFrame]: """Execute any SQL query on AWS Athena and return the results as a Pandas DataFrame. @@ -1002,6 +1017,11 @@ def read_sql_query( Forwarded to `to_pandas` method converting from PyArrow tables to Pandas DataFrame. Valid values include "split_blocks", "self_destruct", "ignore_metadata". e.g. pyarrow_additional_kwargs={'split_blocks': True}. + retreive_workgroup_config + Indicates whether to use the workgroup configuration for the query execution. + If True, the workgroup configuration will be retreived and used to determine the s3 output location, encryption, and kms key. + If False, the s3 output location, encryption, and kms key will not be set and will be determined by the AWS Athena service. + Default is True. Returns ------- @@ -1120,6 +1140,7 @@ def read_sql_query( result_reuse_configuration=result_reuse_configuration, dtype_backend=dtype_backend, client_request_token=client_request_token, + retreive_workgroup_config=retreive_workgroup_config, ) @@ -1386,6 +1407,7 @@ def unload( params: dict[str, Any] | list[str] | None = None, paramstyle: Literal["qmark", "named"] = "named", athena_query_wait_polling_delay: float = _QUERY_WAIT_POLLING_DELAY, + retreive_workgroup_config: bool = True, ) -> _QueryMetadata: """Write query results from a SELECT statement to the specified data format using UNLOAD. @@ -1442,6 +1464,11 @@ def unload( - ``qmark`` athena_query_wait_polling_delay Interval in seconds for how often the function will check if the Athena query has completed. + retreive_workgroup_config + Indicates whether to use the workgroup configuration for the query execution. + If True, the workgroup configuration will be retreived and used to determine the s3 output location, encryption, and kms key. + If False, the s3 output location, encryption, and kms key will not be set and will be determined by the AWS Athena service. + Default is True. Returns ------- @@ -1473,4 +1500,5 @@ def unload( boto3_session=boto3_session, data_source=data_source, execution_params=execution_params, + retreive_workgroup_config=retreive_workgroup_config, ) diff --git a/awswrangler/athena/_read.pyi b/awswrangler/athena/_read.pyi index a3768b851..bcbb1471d 100644 --- a/awswrangler/athena/_read.pyi +++ b/awswrangler/athena/_read.pyi @@ -78,6 +78,7 @@ def read_sql_query( dtype_backend: Literal["numpy_nullable", "pyarrow"] = ..., s3_additional_kwargs: dict[str, Any] | None = ..., pyarrow_additional_kwargs: dict[str, Any] | None = ..., + retreive_workgroup_config: bool = ..., ) -> pd.DataFrame: ... @overload def read_sql_query( @@ -105,6 +106,7 @@ def read_sql_query( dtype_backend: Literal["numpy_nullable", "pyarrow"] = ..., s3_additional_kwargs: dict[str, Any] | None = ..., pyarrow_additional_kwargs: dict[str, Any] | None = ..., + retreive_workgroup_config: bool = ..., ) -> Iterator[pd.DataFrame]: ... @overload def read_sql_query( @@ -132,6 +134,7 @@ def read_sql_query( dtype_backend: Literal["numpy_nullable", "pyarrow"] = ..., s3_additional_kwargs: dict[str, Any] | None = ..., pyarrow_additional_kwargs: dict[str, Any] | None = ..., + retreive_workgroup_config: bool = ..., ) -> pd.DataFrame | Iterator[pd.DataFrame]: ... @overload def read_sql_query( @@ -159,6 +162,7 @@ def read_sql_query( dtype_backend: Literal["numpy_nullable", "pyarrow"] = ..., s3_additional_kwargs: dict[str, Any] | None = ..., pyarrow_additional_kwargs: dict[str, Any] | None = ..., + retreive_workgroup_config: bool = ..., ) -> Iterator[pd.DataFrame]: ... @overload def read_sql_query( @@ -186,6 +190,7 @@ def read_sql_query( dtype_backend: Literal["numpy_nullable", "pyarrow"] = ..., s3_additional_kwargs: dict[str, Any] | None = ..., pyarrow_additional_kwargs: dict[str, Any] | None = ..., + retreive_workgroup_config: bool = ..., ) -> pd.DataFrame | Iterator[pd.DataFrame]: ... @overload def read_sql_table( @@ -210,6 +215,7 @@ def read_sql_table( dtype_backend: Literal["numpy_nullable", "pyarrow"] = ..., s3_additional_kwargs: dict[str, Any] | None = ..., pyarrow_additional_kwargs: dict[str, Any] | None = ..., + retreive_workgroup_config: bool = ..., ) -> pd.DataFrame: ... @overload def read_sql_table( @@ -234,6 +240,7 @@ def read_sql_table( dtype_backend: Literal["numpy_nullable", "pyarrow"] = ..., s3_additional_kwargs: dict[str, Any] | None = ..., pyarrow_additional_kwargs: dict[str, Any] | None = ..., + retreive_workgroup_config: bool = ..., ) -> Iterator[pd.DataFrame]: ... @overload def read_sql_table( @@ -258,6 +265,7 @@ def read_sql_table( dtype_backend: Literal["numpy_nullable", "pyarrow"] = ..., s3_additional_kwargs: dict[str, Any] | None = ..., pyarrow_additional_kwargs: dict[str, Any] | None = ..., + retreive_workgroup_config: bool = ..., ) -> pd.DataFrame | Iterator[pd.DataFrame]: ... @overload def read_sql_table( @@ -282,6 +290,7 @@ def read_sql_table( dtype_backend: Literal["numpy_nullable", "pyarrow"] = ..., s3_additional_kwargs: dict[str, Any] | None = ..., pyarrow_additional_kwargs: dict[str, Any] | None = ..., + retreive_workgroup_config: bool = ..., ) -> Iterator[pd.DataFrame]: ... @overload def read_sql_table( @@ -306,6 +315,7 @@ def read_sql_table( dtype_backend: Literal["numpy_nullable", "pyarrow"] = ..., s3_additional_kwargs: dict[str, Any] | None = ..., pyarrow_additional_kwargs: dict[str, Any] | None = ..., + retreive_workgroup_config: bool = ..., ) -> pd.DataFrame | Iterator[pd.DataFrame]: ... def unload( sql: str, @@ -323,4 +333,5 @@ def unload( params: dict[str, Any] | list[str] | None = ..., paramstyle: Literal["qmark", "named"] = ..., athena_query_wait_polling_delay: float = ..., + retreive_workgroup_config: bool = ..., ) -> _QueryMetadata: ... diff --git a/awswrangler/athena/_utils.py b/awswrangler/athena/_utils.py index 7ff7589cc..b6afbd038 100644 --- a/awswrangler/athena/_utils.py +++ b/awswrangler/athena/_utils.py @@ -140,14 +140,16 @@ def _start_query_execution( return response["QueryExecutionId"] -def _get_workgroup_config(session: boto3.Session | None = None, workgroup: str = "primary") -> _WorkGroupConfig: +def _get_workgroup_config( + session: boto3.Session | None = None, workgroup: str | None = "primary", retreive_workgroup_config: bool = True +) -> _WorkGroupConfig: enforced: bool wg_s3_output: str | None wg_encryption: str | None wg_kms_key: str | None enforced, wg_s3_output, wg_encryption, wg_kms_key = False, None, None, None - if workgroup is not None: + if workgroup is not None and retreive_workgroup_config: res = get_work_group(workgroup=workgroup, boto3_session=session) enforced = res["WorkGroup"]["Configuration"]["EnforceWorkGroupConfiguration"] config: dict[str, Any] = res["WorkGroup"]["Configuration"].get("ResultConfiguration") @@ -481,6 +483,7 @@ def repair_table( kms_key: str | None = None, athena_query_wait_polling_delay: float = _QUERY_WAIT_POLLING_DELAY, boto3_session: boto3.Session | None = None, + retreive_workgroup_config: bool = True, ) -> str: """Run the Hive's metastore consistency check: 'MSCK REPAIR TABLE table;'. @@ -515,6 +518,11 @@ def repair_table( Interval in seconds for how often the function will check if the Athena query has completed. boto3_session The default boto3 session will be used if **boto3_session** receive ``None``. + retreive_workgroup_config + Indicates whether to use the workgroup configuration for the query execution. + If True, the workgroup configuration will be retreived and used to determine the s3 output location, encryption, and kms key. + If False, the s3 output location, encryption, and kms key will not be set and will be determined by the AWS Athena service. + Default is True. Returns ------- @@ -538,6 +546,7 @@ def repair_table( encryption=encryption, kms_key=kms_key, boto3_session=boto3_session, + retreive_workgroup_config=retreive_workgroup_config, ) response: dict[str, Any] = _executions.wait_query( query_execution_id=query_id, @@ -561,6 +570,7 @@ def describe_table( athena_query_wait_polling_delay: float = _QUERY_WAIT_POLLING_DELAY, s3_additional_kwargs: dict[str, Any] | None = None, boto3_session: boto3.Session | None = None, + retreive_workgroup_config: bool = True, ) -> pd.DataFrame: """Show the list of columns, including partition columns: 'DESCRIBE table;'. @@ -593,6 +603,11 @@ def describe_table( e.g. s3_additional_kwargs={'RequestPayer': 'requester'} boto3_session The default boto3 session will be used if **boto3_session** receive ``None``. + retreive_workgroup_config + Indicates whether to use the workgroup configuration for the query execution. + If True, the workgroup configuration will be retreived and used to determine the s3 output location, encryption, and kms key. + If False, the s3 output location, encryption, and kms key will not be set and will be determined by the AWS Athena service. + Default is True. Returns ------- @@ -615,6 +630,7 @@ def describe_table( encryption=encryption, kms_key=kms_key, boto3_session=boto3_session, + retreive_workgroup_config=retreive_workgroup_config, ) query_metadata: _QueryMetadata = _get_query_metadata( query_execution_id=query_id, @@ -643,7 +659,7 @@ def create_ctas_table( bucketing_info: typing.BucketingInfoTuple | None = None, field_delimiter: str | None = None, schema_only: bool = False, - workgroup: str = "primary", + workgroup: str | None = "primary", data_source: str | None = None, encryption: str | None = None, kms_key: str | None = None, @@ -654,6 +670,7 @@ def create_ctas_table( params: dict[str, Any] | list[str] | None = None, paramstyle: Literal["qmark", "named"] = "named", boto3_session: boto3.Session | None = None, + retreive_workgroup_config: bool = True, ) -> dict[str, str | _QueryMetadata]: """Create a new table populated with the results of a SELECT query. @@ -719,6 +736,11 @@ def create_ctas_table( The default is ``named``. boto3_session The default boto3 session will be used if **boto3_session** receive ``None``. + retreive_workgroup_config + Indicates whether to use the workgroup configuration for the query execution. + If True, the workgroup configuration will be retreived and used to determine the s3 output location, encryption, and kms key. + If False, the s3 output location, encryption, and kms key will not be set and will be determined by the AWS Athena service. + Default is True. Returns ------- @@ -783,7 +805,9 @@ def create_ctas_table( fully_qualified_name = f'"{ctas_database}"."{ctas_table}"' - wg_config: _WorkGroupConfig = _get_workgroup_config(session=boto3_session, workgroup=workgroup) + wg_config: _WorkGroupConfig = _get_workgroup_config( + session=boto3_session, workgroup=workgroup, retreive_workgroup_config=retreive_workgroup_config + ) s3_output = _get_s3_output(s3_output=s3_output, wg_config=wg_config, boto3_session=boto3_session) s3_output = s3_output[:-1] if s3_output[-1] == "/" else s3_output # If the workgroup enforces an external location, then it overrides the user supplied argument @@ -891,6 +915,7 @@ def show_create_table( athena_query_wait_polling_delay: float = _QUERY_WAIT_POLLING_DELAY, s3_additional_kwargs: dict[str, Any] | None = None, boto3_session: boto3.Session | None = None, + retreive_workgroup_config: bool = True, ) -> str: """Generate the query that created it: 'SHOW CREATE TABLE table;'. @@ -922,6 +947,11 @@ def show_create_table( e.g. s3_additional_kwargs={'RequestPayer': 'requester'} boto3_session The default boto3 session will be used if **boto3_session** receive ``None``. + retreive_workgroup_config + Indicates whether to use the workgroup configuration for the query execution. + If True, the workgroup configuration will be retreived and used to determine the s3 output location, encryption, and kms key. + If False, the s3 output location, encryption, and kms key will not be set and will be determined by the AWS Athena service. + Default is True. Returns ------- @@ -944,6 +974,7 @@ def show_create_table( encryption=encryption, kms_key=kms_key, boto3_session=boto3_session, + retreive_workgroup_config=retreive_workgroup_config, ) query_metadata: _QueryMetadata = _get_query_metadata( query_execution_id=query_id, diff --git a/awswrangler/athena/_write_iceberg.py b/awswrangler/athena/_write_iceberg.py index 87468f975..c736cb6ed 100644 --- a/awswrangler/athena/_write_iceberg.py +++ b/awswrangler/athena/_write_iceberg.py @@ -259,6 +259,7 @@ def _merge_iceberg( workgroup: str = "primary", encryption: str | None = None, data_source: str | None = None, + retreive_workgroup_config: bool = True, ) -> None: """ Merge iceberg. @@ -297,13 +298,20 @@ def _merge_iceberg( Valid values: [None, 'SSE_S3', 'SSE_KMS']. Notice: 'CSE_KMS' is not supported. data_source : str, optional Data Source / Catalog name. If None, 'AwsDataCatalog' will be used by default. + retreive_workgroup_config: bool, optional + Indicates whether to use the workgroup configuration for the query execution. + If True, the workgroup configuration will be retreived and used to determine the s3 output location, encryption, and kms key. + If False, the s3 output location, encryption, and kms key will not be set and will be determined by the AWS Athena service. + Default is True. Returns ------- None """ - wg_config: _WorkGroupConfig = _get_workgroup_config(session=boto3_session, workgroup=workgroup) + wg_config: _WorkGroupConfig = _get_workgroup_config( + session=boto3_session, workgroup=workgroup, retreive_workgroup_config=retreive_workgroup_config + ) sql_statement: str if merge_cols: @@ -379,6 +387,7 @@ def to_iceberg( # noqa: PLR0913 schema_evolution: bool = False, fill_missing_columns_in_df: bool = True, glue_table_settings: GlueTableSettings | None = None, + retreive_workgroup_config: bool = True, ) -> None: """ Insert into Athena Iceberg table using INSERT INTO ... SELECT. Will create Iceberg table if it does not exist. @@ -461,6 +470,12 @@ def to_iceberg( # noqa: PLR0913 Glue/Athena catalog: Settings for writing to the Glue table. Currently only the 'columns_comments' attribute is supported for this function. Columns comments can only be added with this function when creating a new table. + retreive_workgroup_config: bool = True, + Indicates whether to use the workgroup configuration for the query execution. + If True, the workgroup configuration will be retreived and used to determine the s3 output location, encryption, and kms key. + If False, the s3 output location, encryption, and kms key will not be set and will be determined by the AWS Athena service. + Default is True. + Examples -------- @@ -488,7 +503,9 @@ def to_iceberg( # noqa: PLR0913 ... ) """ - wg_config: _WorkGroupConfig = _get_workgroup_config(session=boto3_session, workgroup=workgroup) + wg_config: _WorkGroupConfig = _get_workgroup_config( + session=boto3_session, workgroup=workgroup, retreive_workgroup_config=retreive_workgroup_config + ) temp_table: str = f"temp_table_{uuid.uuid4().hex}" _validate_args( @@ -584,6 +601,7 @@ def to_iceberg( # noqa: PLR0913 boto3_session=boto3_session, s3_additional_kwargs=s3_additional_kwargs, catalog_id=catalog_id, + retreive_workgroup_config=retreive_workgroup_config, ) # if mode == "overwrite", delete whole data from table (but not table itself) elif mode == "overwrite": @@ -631,6 +649,7 @@ def to_iceberg( # noqa: PLR0913 workgroup=workgroup, encryption=encryption, data_source=data_source, + retreive_workgroup_config=retreive_workgroup_config, ) except Exception as ex: @@ -670,6 +689,7 @@ def delete_from_iceberg_table( boto3_session: boto3.Session | None = None, s3_additional_kwargs: dict[str, Any] | None = None, catalog_id: str | None = None, + retreive_workgroup_config: bool = True, ) -> None: """ Delete rows from an Iceberg table. @@ -714,6 +734,11 @@ def delete_from_iceberg_table( catalog_id The ID of the Data Catalog which contains the database and table. If none is provided, the AWS account ID is used by default. + retreive_workgroup_config: bool = True, + Indicates whether to use the workgroup configuration for the query execution. + If True, the workgroup configuration will be retreived and used to determine the s3 output location, encryption, and kms key. + If False, the s3 output location, encryption, and kms key will not be set and will be determined by the AWS Athena service. + Default is True. Examples -------- @@ -743,7 +768,9 @@ def delete_from_iceberg_table( if not merge_cols: raise exceptions.InvalidArgumentValue("Merge columns must be specified.") - wg_config: _WorkGroupConfig = _get_workgroup_config(session=boto3_session, workgroup=workgroup) + wg_config: _WorkGroupConfig = _get_workgroup_config( + session=boto3_session, workgroup=workgroup, retreive_workgroup_config=retreive_workgroup_config + ) temp_table: str = f"temp_table_{uuid.uuid4().hex}" if not temp_path and not wg_config.s3_output: From 8a02cb794b44a9768199967f19660e206f154150 Mon Sep 17 00:00:00 2001 From: Inbar Shani Date: Tue, 4 Nov 2025 15:03:24 +0200 Subject: [PATCH 2/2] Ignore potery lock --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ac43a087c..4bb80e01e 100644 --- a/.gitignore +++ b/.gitignore @@ -159,6 +159,7 @@ node_modules *.cdk.staging *cdk.out *cdk.context.json +*poetry.lock # ruff .ruff_cache/ \ No newline at end of file