diff --git a/base_requirements.txt b/base_requirements.txt index ac1d1115e4..4781652cda 100644 --- a/base_requirements.txt +++ b/base_requirements.txt @@ -2,6 +2,10 @@ # https://github.com/mozilla/bleach bleach<6.0 +# Python client for Amazon AWS API +# https://github.com/boto/boto3 +boto3 + # The Python web framework on which NetBox is built # https://github.com/django/django Django<4.2 diff --git a/docs/models/core/datasource.md b/docs/models/core/datasource.md index d16abdd100..0e18a2aae1 100644 --- a/docs/models/core/datasource.md +++ b/docs/models/core/datasource.md @@ -14,15 +14,17 @@ The type of data source. Supported options include: * Local directory * git repository +* Amazon S3 bucket ### URL The URL identifying the remote source. Some examples are included below. -| Type | Example URL | -|------|-------------| -| Local | file:///var/my/data/source/ | -| git | https://https://github.com/my-organization/my-repo | +| Type | Example URL | +|-----------|----------------------------------------------------| +| Local | file:///path/to/my/data/ | +| git | https://github.com/my-organization/my-repo | +| Amazon S3 | https://s3.us-east-2.amazonaws.com/my-bucket-name/ | ### Status diff --git a/docs/release-notes/version-3.5.md b/docs/release-notes/version-3.5.md index ac2fab66b4..792a244e43 100644 --- a/docs/release-notes/version-3.5.md +++ b/docs/release-notes/version-3.5.md @@ -10,7 +10,7 @@ The static home view has been replaced with a fully customizable dashboard. User #### Remote Data Sources ([#11558](https://github.com/netbox-community/netbox/issues/11558)) -NetBox now has the ability to synchronize arbitrary data from external sources through the new [DataSource](../models/core/datasource.md) and [DataFile](../models/core/datafile.md) models. Synchronized files are stored in the PostgreSQL database, and may be referenced and consumed by other NetBox models, such as export templates and config contexts. Currently, replication from local filesystem paths and from git repositories is supported, and we expect to add support for additional backends in the near future. +NetBox now has the ability to synchronize arbitrary data from external sources through the new [DataSource](../models/core/datasource.md) and [DataFile](../models/core/datafile.md) models. Synchronized files are stored in the PostgreSQL database, and may be referenced and consumed by other NetBox models, such as export templates and config contexts. Currently, replication from local filesystem paths, git repositories, and Amazon S3 buckets is supported, and we expect to introduce additional backends in the near future. #### Configuration Template Rendering ([#11559](https://github.com/netbox-community/netbox/issues/11559)) diff --git a/netbox/core/choices.py b/netbox/core/choices.py index 6927c83fba..be33afabae 100644 --- a/netbox/core/choices.py +++ b/netbox/core/choices.py @@ -10,10 +10,12 @@ from utilities.choices import ChoiceSet class DataSourceTypeChoices(ChoiceSet): LOCAL = 'local' GIT = 'git' + AMAZON_S3 = 'amazon-s3' CHOICES = ( (LOCAL, _('Local'), 'gray'), (GIT, _('Git'), 'blue'), + (AMAZON_S3, _('Amazon S3'), 'blue'), ) diff --git a/netbox/core/data_backends.py b/netbox/core/data_backends.py index 5d0e805842..1160b87eff 100644 --- a/netbox/core/data_backends.py +++ b/netbox/core/data_backends.py @@ -1,9 +1,14 @@ import logging +import os +import re import subprocess import tempfile from contextlib import contextmanager +from pathlib import Path from urllib.parse import quote, urlunparse, urlparse +import boto3 +from botocore.config import Config as Boto3Config from django import forms from django.conf import settings from django.utils.translation import gettext as _ @@ -115,3 +120,70 @@ class GitBackend(DataBackend): yield local_path.name local_path.cleanup() + + +@register_backend(DataSourceTypeChoices.AMAZON_S3) +class S3Backend(DataBackend): + parameters = { + 'aws_access_key_id': forms.CharField( + label=_('AWS access key ID'), + widget=forms.TextInput(attrs={'class': 'form-control'}) + ), + 'aws_secret_access_key': forms.CharField( + label=_('AWS secret access key'), + widget=forms.TextInput(attrs={'class': 'form-control'}) + ), + } + + REGION_REGEX = r's3\.([a-z0-9-]+)\.amazonaws\.com' + + @contextmanager + def fetch(self): + local_path = tempfile.TemporaryDirectory() + + # Build the S3 configuration + s3_config = Boto3Config( + proxies=settings.HTTP_PROXIES, + ) + + # Initialize the S3 resource and bucket + aws_access_key_id = self.params.get('aws_access_key_id') + aws_secret_access_key = self.params.get('aws_secret_access_key') + s3 = boto3.resource( + 's3', + region_name=self._region_name, + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + config=s3_config + ) + bucket = s3.Bucket(self._bucket_name) + + # Download all files within the specified path + for obj in bucket.objects.filter(Prefix=self._remote_path): + local_filename = os.path.join(local_path.name, obj.key) + # Build local path + Path(os.path.dirname(local_filename)).mkdir(parents=True, exist_ok=True) + bucket.download_file(obj.key, local_filename) + + yield local_path.name + + local_path.cleanup() + + @property + def _region_name(self): + domain = urlparse(self.url).netloc + if m := re.match(self.REGION_REGEX, domain): + return m.group(1) + return None + + @property + def _bucket_name(self): + url_path = urlparse(self.url).path.lstrip('/') + return url_path.split('/')[0] + + @property + def _remote_path(self): + url_path = urlparse(self.url).path.lstrip('/') + if '/' in url_path: + return url_path.split('/', 1)[1] + return '' diff --git a/requirements.txt b/requirements.txt index c66477589b..d5f495b7ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ bleach==5.0.1 +boto3==1.26.91 Django==4.1.7 django-cors-headers==3.14.0 django-debug-toolbar==3.8.1