Skip to content

Commit

Permalink
Prepare for streaming downloads
Browse files Browse the repository at this point in the history
pydataverse cannot do it, so this won't have much of an effect

Ping #199
  • Loading branch information
mih committed Mar 14, 2023
1 parent 8d128b5 commit e0a2fc5
Showing 1 changed file with 8 additions and 1 deletion.
9 changes: 8 additions & 1 deletion datalad_dataverse/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,11 +114,18 @@ def is_released_file(self, fid: int) -> bool:
or fid in self.files_old.keys())

def download_file(self, fid: int, path: Path):
# pydataverse does not support streaming downloads
# https://github.com/gdcc/pyDataverse/issues/49
# the code below is nevertheless readied for such a
# scenario
response = self.data_access_api.get_datafile(fid)
# http error handling
response.raise_for_status()
with path.open("wb") as f:
f.write(response.content)
# `chunk_size=None` means
# "read data in whatever size the chunks are received"
for chunk in response.iter_content(chunk_size=None):
f.write(chunk)

def remove_file(self, fid: int):
status = delete_request(
Expand Down

0 comments on commit e0a2fc5

Please sign in to comment.