Merge pull request #464 from dandi/remove-metadata-tables

Remove metadata tables

dandiapi/api/admin.py

@@ -1,15 +1,7 @@
 from django.contrib import admin
 from guardian.admin import GuardedModelAdmin
 
-from dandiapi.api.models import (
-    Asset,
-    AssetBlob,
-    AssetMetadata,
-    Dandiset,
-    Upload,
-    Version,
-    VersionMetadata,
-)
+from dandiapi.api.models import Asset, AssetBlob, Dandiset, Upload, Version
 
 
 @admin.register(Dandiset)
@@ -18,12 +10,6 @@ class DandisetAdmin(GuardedModelAdmin):
     readonly_fields = ['identifier', 'created']
 
 
-@admin.register(VersionMetadata)
-class VersionMetadataAdmin(admin.ModelAdmin):
-    list_display = ['id', 'name', 'metadata', 'references', 'modified', 'created']
-    list_display_links = ['id', 'name', 'metadata']
-
-
 class AssetInline(admin.TabularInline):
     model = Asset.versions.through
 
@@ -41,12 +27,6 @@ class AssetBlobAdmin(admin.ModelAdmin):
     list_display_links = ['id', 'blob_id']
 
 
-@admin.register(AssetMetadata)
-class AssetMetadataAdmin(admin.ModelAdmin):
-    list_display = ['id', 'metadata', 'references', 'modified', 'created']
-    list_display_links = ['id', 'metadata']
-
-
 class AssetBlobInline(admin.TabularInline):
     model = AssetBlob
 

dandiapi/api/doi.py

@@ -20,7 +20,7 @@ def _generate_doi_data(version: Version):
     dandiset_id = version.dandiset.identifier
     version_id = version.version
     doi = f'{prefix}/dandi.{dandiset_id}/{version_id}'
-    metadata = version.metadata.metadata
+    metadata = version.metadata
     metadata['doi'] = doi
     return (doi, to_datacite(metadata, publish=publish))
 

dandiapi/api/management/commands/migrate_version_metadata.py

@@ -13,7 +13,7 @@ def migrate_version_metadata(to_version: str):
     for version in Version.objects.filter(version='draft'):
         print(f'Migrating {version.dandiset.identifier}/{version.version}')
 
-        metadata = version.metadata.metadata
+        metadata = version.metadata
         # If there is no schemaVersion, assume the most recent
         if 'schemaVersion' not in metadata:
             metadata['schemaVersion'] = settings.DANDI_SCHEMA_VERSION
@@ -27,7 +27,7 @@ def migrate_version_metadata(to_version: str):
 
         new: VersionMetadata
         new, created = VersionMetadata.objects.get_or_create(
-            name=version.metadata.name,
+            name=version.name,
             metadata=metanew,
         )
         if created:

dandiapi/api/manifests.py

@@ -73,31 +73,31 @@ def _write_manifest_file(path: str, metadata, logger):
 def write_dandiset_jsonld(version: Version, logger=None):
     _write_manifest_file(
         dandiset_jsonld_path(version),
-        JSONRenderer().render(version.metadata.metadata),
+        JSONRenderer().render(version.metadata),
         logger,
     )
 
 
 def write_assets_jsonld(version: Version, logger=None):
     _write_manifest_file(
         assets_jsonld_path(version),
-        JSONRenderer().render([asset.metadata.metadata for asset in version.assets.all()]),
+        JSONRenderer().render([asset.metadata for asset in version.assets.all()]),
         logger,
     )
 
 
 def write_dandiset_yaml(version: Version, logger=None):
     _write_manifest_file(
         dandiset_yaml_path(version),
-        YAMLRenderer().render(version.metadata.metadata),
+        YAMLRenderer().render(version.metadata),
         logger,
     )
 
 
 def write_assets_yaml(version: Version, logger=None):
     _write_manifest_file(
         assets_yaml_path(version),
-        YAMLRenderer().render([asset.metadata.metadata for asset in version.assets.all()]),
+        YAMLRenderer().render([asset.metadata for asset in version.assets.all()]),
         logger,
     )
 
@@ -107,10 +107,10 @@ def write_collection_jsonld(version: Version, logger=None):
         collection_jsonld_path(version),
         JSONRenderer().render(
             {
-                '@context': version.metadata.metadata['@context'],
-                'id': version.metadata.metadata['id'],
+                '@context': version.metadata['@context'],
+                'id': version.metadata['id'],
                 '@type': 'prov:Collection',
-                'hasMember': [asset.metadata.metadata['id'] for asset in version.assets.all()],
+                'hasMember': [asset.metadata['id'] for asset in version.assets.all()],
             }
         ),
         logger,

dandiapi/api/migrations/0016_remove_metadata_tables.py

@@ -0,0 +1,88 @@
+# Generated by Django 3.2 on 2021-08-09 18:00
+
+import django.contrib.postgres.indexes
+from django.db import migrations, models
+
+
+def migrate_metadata(apps, schema_editor):
+    Version = apps.get_model('api', 'Version')  # noqa: N806
+    Asset = apps.get_model('api', 'Asset')  # noqa: N806
+
+    versions = Version.objects.all()
+    for version in versions:
+        version.name = version.old_metadata.name
+        version.metadata = version.old_metadata.metadata
+    Version.objects.bulk_update(versions, ['name', 'metadata'])
+
+    assets = Asset.objects.all()
+    for asset in assets:
+        asset.metadata = asset.old_metadata.metadata
+    Asset.objects.bulk_update(assets, ['metadata'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0015_validation_fields'),
+    ]
+
+    operations = [
+        # Move the original metadata fields to old_metadata
+        migrations.RenameField(
+            model_name='asset',
+            old_name='metadata',
+            new_name='old_metadata',
+        ),
+        migrations.RenameField(
+            model_name='version',
+            old_name='metadata',
+            new_name='old_metadata',
+        ),
+        # Create metadata field on Asset
+        migrations.AddField(
+            model_name='asset',
+            name='metadata',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        # Create name and metadata fields on Version
+        migrations.AddField(
+            model_name='version',
+            name='name',
+            field=models.CharField(default='temporary_name', max_length=300),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='version',
+            name='metadata',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.AddIndex(
+            model_name='version',
+            index=django.contrib.postgres.indexes.HashIndex(
+                fields=['metadata'], name='api_version_metadat_f0b8f8_hash'
+            ),
+        ),
+        migrations.AddIndex(
+            model_name='version',
+            index=django.contrib.postgres.indexes.HashIndex(
+                fields=['name'], name='api_version_name_5d8af6_hash'
+            ),
+        ),
+        # Move the data from the old_metadata fields to the new metadata fields
+        migrations.RunPython(migrate_metadata),
+        # Delete the old_metadata fields
+        migrations.RemoveField(
+            model_name='Asset',
+            name='old_metadata',
+        ),
+        migrations.RemoveField(
+            model_name='Version',
+            name='old_metadata',
+        ),
+        migrations.DeleteModel(
+            name='AssetMetadata',
+        ),
+        migrations.DeleteModel(
+            name='VersionMetadata',
+        ),
+    ]

dandiapi/api/models/__init__.py

@@ -1,16 +1,14 @@
-from .asset import Asset, AssetBlob, AssetMetadata
+from .asset import Asset, AssetBlob
 from .dandiset import Dandiset
 from .oauth import StagingApplication
 from .upload import Upload
-from .version import Version, VersionMetadata
+from .version import Version
 
 __all__ = [
     'Asset',
     'AssetBlob',
-    'AssetMetadata',
     'Dandiset',
     'StagingApplication',
     'Upload',
     'Version',
-    'VersionMetadata',
 ]

dandiapi/api/models/asset.py

@@ -75,18 +75,7 @@ def __str__(self) -> str:
         return self.blob.name
 
 
-class AssetMetadata(PublishableMetadataMixin, TimeStampedModel):
-    metadata = models.JSONField(blank=True, unique=True, default=dict)
-
-    @property
-    def references(self) -> int:
-        return self.assets.count()
-
-    def __str__(self) -> str:
-        return str(self.metadata)
-
-
-class Asset(TimeStampedModel):
+class Asset(PublishableMetadataMixin, TimeStampedModel):
     UUID_REGEX = r'[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}'
 
     class Status(models.TextChoices):
@@ -98,7 +87,7 @@ class Status(models.TextChoices):
     asset_id = models.UUIDField(unique=True, default=uuid.uuid4)
     path = models.CharField(max_length=512)
     blob = models.ForeignKey(AssetBlob, related_name='assets', on_delete=models.CASCADE)
-    metadata = models.ForeignKey(AssetMetadata, related_name='assets', on_delete=models.CASCADE)
+    metadata = models.JSONField(blank=True, default=dict)
     versions = models.ManyToManyField(Version, related_name='assets')
     status = models.CharField(
         max_length=10,
@@ -132,7 +121,7 @@ def _populate_metadata(self):
         blob_url = self.blob.s3_url
 
         metadata = {
-            **self.metadata.metadata,
+            **self.metadata,
             'id': f'dandiasset:{self.asset_id}',
             'path': self.path,
             'identifier': str(self.asset_id),
@@ -152,14 +141,11 @@ def published_metadata(self):
         """Generate the metadata of this asset as if it were being published."""
         now = datetime.datetime.utcnow()
         # Inject the publishedBy and datePublished fields
-        published_metadata, _ = AssetMetadata.objects.get_or_create(
-            metadata={
-                **self.metadata.metadata,
-                'publishedBy': self.metadata.published_by(now),
-                'datePublished': now.isoformat(),
-            },
-        )
-        return published_metadata
+        return {
+            **self.metadata,
+            'publishedBy': self.published_by(now),
+            'datePublished': now.isoformat(),
+        }
 
     def publish(self):
         """
@@ -173,11 +159,7 @@ def publish(self):
         self.published = True
 
     def save(self, *args, **kwargs):
-        metadata = self._populate_metadata()
-        new, created = AssetMetadata.objects.get_or_create(metadata=metadata)
-        if created:
-            new.save()
-        self.metadata = new
+        self.metadata = self._populate_metadata()
         super().save(*args, **kwargs)
 
     @classmethod

dandiapi/api/models/metadata.py

@@ -8,7 +8,8 @@ def published_by(self, now: datetime.datetime):
             'id': uuid4().urn,
             'name': 'DANDI publish',
             'startDate': now.isoformat(),
-            # TODO endDate needs to be defined before publish is complete
+            # TODO https://github.com/dandi/dandi-api/issues/465
+            # endDate needs to be defined before publish is complete
             'endDate': now.isoformat(),
             'wasAssociatedWith': [
                 {