Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove pinning on django-storages[boto3] #1717

Merged
merged 2 commits into from
Oct 18, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions dandiapi/api/copy.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@
from dandiapi.api.storage import get_boto_client

try:
from storages.backends.s3boto3 import S3Boto3Storage
from storages.backends.s3 import S3Storage
except ImportError:
# This should only be used for type interrogation, never instantiation
S3Boto3Storage = type('FakeS3Boto3Storage', (), {})
S3Storage = type('FakeS3Storage', (), {})
try:
from minio_storage.storage import MinioStorage
except ImportError:
Expand Down
4 changes: 2 additions & 2 deletions dandiapi/api/management/commands/cleanup_blobs.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
from django.conf import settings
import djclick as click
from storages.backends.s3boto3 import S3Boto3Storage
from storages.backends.s3 import S3Storage

from dandiapi.api.models.upload import AssetBlob

BUCKET = settings.DANDI_DANDISETS_BUCKET_NAME


def s3_client():
storage = S3Boto3Storage(bucket_name=BUCKET)
storage = S3Storage(bucket_name=BUCKET)
return storage.connection.meta.client


Expand Down
16 changes: 8 additions & 8 deletions dandiapi/api/storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from minio_storage.storage import MinioStorage, create_minio_client_from_settings
from s3_file_field._multipart_boto3 import Boto3MultipartManager
from s3_file_field._multipart_minio import MinioMultipartManager
from storages.backends.s3boto3 import S3Boto3Storage
from storages.backends.s3 import S3Storage


class ChecksumCalculatorFile:
Expand Down Expand Up @@ -95,7 +95,7 @@ def __init__(self, *args, **kwargs):
class VerbatimNameStorageMixin:
"""A Storage mixin, storing files without transforming their original filename."""

# The basic S3Boto3Storage does not implement generate_filename or get_valid_name,
# The basic S3Storage does not implement generate_filename or get_valid_name,
# so upon FileField save, the following call stack normally occurs:
# FieldFile.save
# FileField.generate_filename
Expand All @@ -110,7 +110,7 @@ def generate_filename(self, filename: str) -> str:
return filename


class TimeoutS3Boto3Storage(S3Boto3Storage):
class TimeoutS3Storage(S3Storage):
"""Override boto3 default timeout values."""

def __init__(self, **settings):
Expand All @@ -121,7 +121,7 @@ def __init__(self, **settings):
)


class VerbatimNameS3Storage(VerbatimNameStorageMixin, TimeoutS3Boto3Storage):
class VerbatimNameS3Storage(VerbatimNameStorageMixin, TimeoutS3Storage):
@property
def multipart_manager(self):
return DandiBoto3MultipartManager(self)
Expand Down Expand Up @@ -246,14 +246,14 @@ def create_s3_storage(bucket_name: str) -> Storage:
"""
Return a new Storage instance, compatible with the default Storage class.

This abstracts over differences between S3Boto3Storage and MinioStorage,
This abstracts over differences between S3Storage and MinioStorage,
allowing either to be used as an additional non-default Storage.
"""
# For production, calling django.core.files.storage.get_storage_class is fine
# to return the storage class of S3Boto3Storage.
# to return the storage class of S3Storage.
default_storage_class = get_storage_class()

if issubclass(default_storage_class, S3Boto3Storage):
if issubclass(default_storage_class, S3Storage):
storage = VerbatimNameS3Storage(bucket_name=bucket_name)
# Required to upload to the sponsored bucket
storage.default_acl = 'bucket-owner-full-control'
Expand All @@ -280,7 +280,7 @@ def create_s3_storage(bucket_name: str) -> Storage:
storage = VerbatimNameMinioStorage(
bucket_name=bucket_name,
base_url=base_url,
# All S3Boto3Storage URLs are presigned, and the bucket typically is not public
# All S3Storage URLs are presigned, and the bucket typically is not public
presign_urls=True,
auto_create_bucket=True,
auto_create_policy=True,
Expand Down
4 changes: 2 additions & 2 deletions dandiapi/api/views/asset.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@
from dandiapi.zarr.models import ZarrArchive

try:
from storages.backends.s3boto3 import S3Boto3Storage
from storages.backends.s3 import S3Storage
except ImportError:
# This should only be used for type interrogation, never instantiation
S3Boto3Storage = type('FakeS3Boto3Storage', (), {})
S3Storage = type('FakeS3Storage', (), {})
try:
from minio_storage.storage import MinioStorage
except ImportError:
Expand Down
10 changes: 5 additions & 5 deletions dandiapi/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import pytest
from pytest_factoryboy import register
from rest_framework.test import APIClient
from storages.backends.s3boto3 import S3Boto3Storage
from storages.backends.s3 import S3Storage

from dandiapi.api.storage import create_s3_storage
from dandiapi.api.tests.factories import (
Expand Down Expand Up @@ -84,7 +84,7 @@ def authenticated_api_client(user) -> APIClient:
# storage fixtures are copied from django-s3-file-field test fixtures


def base_s3boto3_storage_factory(bucket_name: str) -> 'S3Boto3Storage':
def base_s3boto3_storage_factory(bucket_name: str) -> 'S3Storage':
return create_s3_storage(bucket_name)


Expand All @@ -109,12 +109,12 @@ def embargoed_minio_storage_factory() -> MinioStorage:


@pytest.fixture
def s3boto3_storage() -> 'S3Boto3Storage':
def s3boto3_storage() -> 'S3Storage':
return s3boto3_storage_factory()


@pytest.fixture
def embargoed_s3boto3_storage() -> 'S3Boto3Storage':
def embargoed_s3boto3_storage() -> 'S3Storage':
return s3boto3_storage_factory()


Expand All @@ -132,7 +132,7 @@ def embargoed_minio_storage() -> MinioStorage:
def storage(request, settings) -> Storage:
storage_factory = request.param
if storage_factory == s3boto3_storage_factory:
settings.DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
settings.DEFAULT_FILE_STORAGE = 'storages.backends.s3.S3Storage'
settings.AWS_S3_ACCESS_KEY_ID = settings.MINIO_STORAGE_ACCESS_KEY
settings.AWS_S3_SECRET_ACCESS_KEY = settings.MINIO_STORAGE_SECRET_KEY
settings.AWS_S3_REGION_NAME = 'test-region'
Expand Down
3 changes: 1 addition & 2 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,7 @@
'django-composed-configuration[prod]>=0.22.0',
# pin directly to a version since we're extending the private multipart interface
'django-s3-file-field[boto3]==0.3.2',
# TODO: unpin this when dandi is updating for breaking changes
'django-storages[boto3]<=1.13.2',
'django-storages[s3]>=1.14.2',
'gunicorn',
# Development-only, but required
# TODO: starting with v0.5.0, django-minio-storage requires v7
Expand Down