Skip to content

Commit

Permalink
Pass custom delimiter from Table.from_csv to BigQuery load
Browse files Browse the repository at this point in the history
If our source table is loaded from CSV with no transformations, the
original source file will be directly loaded to GCS, we may need to
pass along a custom delimiter to BigQuery.
  • Loading branch information
austinweisgrau committed Jun 24, 2024
1 parent 061bb89 commit 5222c9c
Showing 1 changed file with 10 additions and 0 deletions.
10 changes: 10 additions & 0 deletions parsons/google/google_bigquery.py
Original file line number Diff line number Diff line change
Expand Up @@ -782,6 +782,15 @@ def copy(

self._validate_copy_inputs(if_exists=if_exists, data_type=data_type)

# If our source table is loaded from CSV with no transformations
# The original source file will be directly loaded to GCS
# We may need to pass along a custom delimiter to BigQuery
# Otherwise we use the default comma
if isinstance(tbl.table, petl.io.csv_py3.CSVView):
csv_delimiter = tbl.table.csvargs.get("delimiter", ",")
else:
csv_delimiter = ","

job_config = self._process_job_config(
job_config=job_config,
destination_table_name=table_name,
Expand All @@ -796,6 +805,7 @@ def copy(
allow_jagged_rows=allow_jagged_rows,
quote=quote,
custom_schema=schema,
csv_delimiter=csv_delimiter,
)

# Reorder schema to match table to ensure compatibility
Expand Down

0 comments on commit 5222c9c

Please sign in to comment.