refactor(api): migrate inventory processing to background worker
!39 #76
This commit is contained in:
316
app/api/tasks.py
Normal file
316
app/api/tasks.py
Normal file
@ -0,0 +1,316 @@
|
||||
import json
|
||||
import re
|
||||
|
||||
from django.utils import timezone
|
||||
|
||||
from celery import shared_task, current_task
|
||||
from celery.utils.log import get_task_logger
|
||||
from celery import states
|
||||
|
||||
from access.models import Organization
|
||||
|
||||
from api.serializers.inventory import Inventory
|
||||
|
||||
from itam.models.device import Device, DeviceType, DeviceOperatingSystem, DeviceSoftware
|
||||
from itam.models.operating_system import OperatingSystem, OperatingSystemVersion
|
||||
from itam.models.software import Software, SoftwareCategory, SoftwareVersion
|
||||
|
||||
from settings.models.app_settings import AppSettings
|
||||
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
@shared_task(bind=True)
|
||||
def process_inventory(self, data, organization: int):
|
||||
|
||||
device = None
|
||||
device_operating_system = None
|
||||
operating_system = None
|
||||
operating_system_version = None
|
||||
|
||||
try:
|
||||
|
||||
logger.info('Begin Processing Inventory')
|
||||
|
||||
data = json.loads(data)
|
||||
data = Inventory(data)
|
||||
|
||||
organization = Organization.objects.get(id=organization)
|
||||
|
||||
if Device.objects.filter(slug=str(data.details.name).lower()).exists():
|
||||
|
||||
device = Device.objects.get(slug=str(data.details.name).lower())
|
||||
|
||||
# device = self.obj
|
||||
|
||||
|
||||
app_settings = AppSettings.objects.get(owner_organization = None)
|
||||
|
||||
if not device: # Create the device
|
||||
|
||||
device_serial_number = None
|
||||
|
||||
if not data.details.serial_number and str(data.details.serial_number).lower() != 'na':
|
||||
|
||||
device_serial_number = data.details.serial_number
|
||||
|
||||
device = Device.objects.create(
|
||||
name = data.details.name,
|
||||
device_type = None,
|
||||
serial_number = data.details.serial_number,
|
||||
uuid = data.details.uuid,
|
||||
organization = organization,
|
||||
)
|
||||
|
||||
|
||||
if device:
|
||||
|
||||
logger.info(f"Device: {device.name}, Serial: {device.serial_number}, UUID: {device.uuid}")
|
||||
|
||||
if not device.uuid and data.details.uuid:
|
||||
|
||||
device.uuid = data.details.uuid
|
||||
|
||||
device.save()
|
||||
|
||||
|
||||
if not device.serial_number and data.details.serial_number:
|
||||
|
||||
device.serial_number = data.details.serial_number
|
||||
|
||||
device.save()
|
||||
|
||||
|
||||
if OperatingSystem.objects.filter( slug=data.operating_system.name ).exists():
|
||||
|
||||
operating_system = OperatingSystem.objects.get( slug=data.operating_system.name )
|
||||
|
||||
else: # Create Operating System
|
||||
|
||||
operating_system = OperatingSystem.objects.create(
|
||||
name = data.operating_system.name,
|
||||
organization = organization,
|
||||
is_global = True
|
||||
)
|
||||
|
||||
|
||||
if OperatingSystemVersion.objects.filter( name=data.operating_system.version_major, operating_system=operating_system ).exists():
|
||||
|
||||
operating_system_version = OperatingSystemVersion.objects.get(
|
||||
organization = organization,
|
||||
is_global = True,
|
||||
name = data.operating_system.version_major,
|
||||
operating_system = operating_system
|
||||
)
|
||||
|
||||
else: # Create Operating System Version
|
||||
|
||||
operating_system_version = OperatingSystemVersion.objects.create(
|
||||
organization = organization,
|
||||
is_global = True,
|
||||
name = data.operating_system.version_major,
|
||||
operating_system = operating_system,
|
||||
)
|
||||
|
||||
|
||||
if DeviceOperatingSystem.objects.filter( version=data.operating_system.version, device=device, operating_system_version=operating_system_version ).exists():
|
||||
|
||||
device_operating_system = DeviceOperatingSystem.objects.get(
|
||||
device=device,
|
||||
version = data.operating_system.version,
|
||||
operating_system_version = operating_system_version,
|
||||
)
|
||||
|
||||
if not device_operating_system.installdate: # Only update install date if empty
|
||||
|
||||
device_operating_system.installdate = timezone.now()
|
||||
|
||||
device_operating_system.save()
|
||||
|
||||
else: # Create Operating System Version
|
||||
|
||||
device_operating_system = DeviceOperatingSystem.objects.create(
|
||||
organization = organization,
|
||||
device=device,
|
||||
version = data.operating_system.version,
|
||||
operating_system_version = operating_system_version,
|
||||
installdate = timezone.now()
|
||||
)
|
||||
|
||||
|
||||
if app_settings.software_is_global:
|
||||
|
||||
software_organization = app_settings.global_organization
|
||||
|
||||
else:
|
||||
|
||||
software_organization = device.organization
|
||||
|
||||
|
||||
if app_settings.software_categories_is_global:
|
||||
|
||||
software_category_organization = app_settings.global_organization
|
||||
|
||||
else:
|
||||
|
||||
software_category_organization = device.organization
|
||||
|
||||
inventoried_software: list = []
|
||||
|
||||
for inventory in list(data.software):
|
||||
|
||||
software = None
|
||||
software_category = None
|
||||
software_version = None
|
||||
|
||||
device_software = None
|
||||
|
||||
software_category = SoftwareCategory.objects.filter( name = inventory.category )
|
||||
|
||||
|
||||
if software_category.exists():
|
||||
|
||||
software_category = SoftwareCategory.objects.get(
|
||||
name = inventory.category
|
||||
)
|
||||
|
||||
else: # Create Software Category
|
||||
|
||||
software_category = SoftwareCategory.objects.create(
|
||||
organization = software_category_organization,
|
||||
is_global = True,
|
||||
name = inventory.category,
|
||||
)
|
||||
|
||||
|
||||
if software_category.name == inventory.category:
|
||||
|
||||
if Software.objects.filter( name = inventory.name ).exists():
|
||||
|
||||
software = Software.objects.get(
|
||||
name = inventory.name
|
||||
)
|
||||
|
||||
if not software.category:
|
||||
|
||||
software.category = software_category
|
||||
software.save()
|
||||
|
||||
else: # Create Software
|
||||
|
||||
software = Software.objects.create(
|
||||
organization = software_organization,
|
||||
is_global = True,
|
||||
name = inventory.name,
|
||||
category = software_category,
|
||||
)
|
||||
|
||||
|
||||
if software.name == inventory.name:
|
||||
|
||||
pattern = r"^(\d+:)?(?P<semver>\d+\.\d+(\.\d+)?)"
|
||||
|
||||
semver = re.search(pattern, str(inventory.version), re.DOTALL)
|
||||
|
||||
|
||||
if semver:
|
||||
|
||||
semver = semver['semver']
|
||||
|
||||
else:
|
||||
semver = inventory.version
|
||||
|
||||
|
||||
if SoftwareVersion.objects.filter( name = semver, software = software ).exists():
|
||||
|
||||
software_version = SoftwareVersion.objects.get(
|
||||
name = semver,
|
||||
software = software,
|
||||
)
|
||||
|
||||
else: # Create Software Category
|
||||
|
||||
software_version = SoftwareVersion.objects.create(
|
||||
organization = organization,
|
||||
is_global = True,
|
||||
name = semver,
|
||||
software = software,
|
||||
)
|
||||
|
||||
|
||||
if software_version.name == semver:
|
||||
|
||||
if DeviceSoftware.objects.filter( software = software, device=device ).exists():
|
||||
|
||||
device_software = DeviceSoftware.objects.get(
|
||||
device = device,
|
||||
software = software
|
||||
)
|
||||
|
||||
logger.debug(f"Select Existing Device Software: {device_software.software.name}")
|
||||
|
||||
else: # Create Software
|
||||
|
||||
device_software = DeviceSoftware.objects.create(
|
||||
organization = organization,
|
||||
is_global = True,
|
||||
installedversion = software_version,
|
||||
software = software,
|
||||
device = device,
|
||||
action=None
|
||||
)
|
||||
|
||||
|
||||
logger.debug(f"Create Device Software: {device_software.software.name}")
|
||||
|
||||
|
||||
if device_software: # Update the Inventoried software
|
||||
|
||||
inventoried_software += [ device_software.id ]
|
||||
|
||||
|
||||
if not device_software.installed: # Only update install date if blank
|
||||
|
||||
device_software.installed = timezone.now()
|
||||
|
||||
device_software.save()
|
||||
|
||||
logger.debug(f"Update Device Software (installed): {device_software.software.name}")
|
||||
|
||||
|
||||
if device_software.installedversion.name != software_version.name:
|
||||
|
||||
device_software.installedversion = software_version
|
||||
|
||||
device_software.save()
|
||||
|
||||
logger.debug(f"Update Device Software (installedversion): {device_software.software.name}")
|
||||
|
||||
for not_installed in DeviceSoftware.objects.filter( device=device ):
|
||||
|
||||
if not_installed.id not in inventoried_software:
|
||||
|
||||
not_installed.delete()
|
||||
|
||||
logger.debug(f"Remove Device Software: {not_installed.software.name}")
|
||||
|
||||
|
||||
if device and operating_system and operating_system_version and device_operating_system:
|
||||
|
||||
|
||||
device.inventorydate = timezone.now()
|
||||
|
||||
device.save()
|
||||
|
||||
|
||||
logger.info('Finish Processing Inventory')
|
||||
|
||||
return str('finished...')
|
||||
|
||||
except Exception as e:
|
||||
|
||||
logger.critical('Exception')
|
||||
|
||||
raise Exception(e)
|
||||
|
||||
return str(f'Exception Occured: {e}')
|
||||
@ -1,32 +1,25 @@
|
||||
# from django.contrib.auth.mixins import PermissionRequiredMixin, LoginRequiredMixin
|
||||
import json
|
||||
import re
|
||||
|
||||
from django.core.exceptions import ValidationError, PermissionDenied
|
||||
from django.http import Http404, JsonResponse
|
||||
from django.utils import timezone
|
||||
|
||||
from drf_spectacular.utils import extend_schema, OpenApiExample, OpenApiTypes, OpenApiResponse, OpenApiParameter
|
||||
from drf_spectacular.utils import extend_schema, OpenApiResponse
|
||||
|
||||
from rest_framework import generics, views
|
||||
from rest_framework.response import Response
|
||||
|
||||
from access.mixin import OrganizationMixin
|
||||
from access.models import Organization
|
||||
|
||||
from api.views.mixin import OrganizationPermissionAPI
|
||||
from api.serializers.itam.inventory import InventorySerializer
|
||||
from api.serializers.inventory import Inventory
|
||||
|
||||
from core.http.common import Http
|
||||
|
||||
from itam.models.device import Device, DeviceType, DeviceOperatingSystem, DeviceSoftware
|
||||
from itam.models.operating_system import OperatingSystem, OperatingSystemVersion
|
||||
from itam.models.software import Software, SoftwareCategory, SoftwareVersion
|
||||
from itam.models.device import Device
|
||||
|
||||
from settings.models.app_settings import AppSettings
|
||||
from settings.models.user_settings import UserSettings
|
||||
|
||||
from api.tasks import process_inventory
|
||||
|
||||
|
||||
|
||||
class InventoryPermissions(OrganizationPermissionAPI):
|
||||
@ -102,264 +95,9 @@ this setting populated, no device will be created and the endpoint will return H
|
||||
|
||||
raise Http404
|
||||
|
||||
device_operating_system = None
|
||||
operating_system = None
|
||||
operating_system_version = None
|
||||
task = process_inventory.delay(request.body, self.default_organization.id)
|
||||
|
||||
|
||||
app_settings = AppSettings.objects.get(owner_organization = None)
|
||||
|
||||
if not device: # Create the device
|
||||
|
||||
device = Device.objects.create(
|
||||
name = data.details.name,
|
||||
device_type = None,
|
||||
serial_number = data.details.serial_number,
|
||||
uuid = data.details.uuid,
|
||||
organization = self.default_organization,
|
||||
)
|
||||
|
||||
status = Http.Status.CREATED
|
||||
|
||||
|
||||
if not device.uuid and data.details.uuid:
|
||||
|
||||
device.uuid = data.details.uuid
|
||||
|
||||
device.save()
|
||||
|
||||
|
||||
if not device.serial_number and data.details.serial_number:
|
||||
|
||||
device.serial_number = data.details.serial_number
|
||||
|
||||
device.save()
|
||||
|
||||
|
||||
if OperatingSystem.objects.filter( slug=data.operating_system.name ).exists():
|
||||
|
||||
operating_system = OperatingSystem.objects.get( slug=data.operating_system.name )
|
||||
|
||||
else: # Create Operating System
|
||||
|
||||
operating_system = OperatingSystem.objects.create(
|
||||
name = data.operating_system.name,
|
||||
organization = self.default_organization,
|
||||
is_global = True
|
||||
)
|
||||
|
||||
|
||||
if OperatingSystemVersion.objects.filter( name=data.operating_system.version_major, operating_system=operating_system ).exists():
|
||||
|
||||
operating_system_version = OperatingSystemVersion.objects.get(
|
||||
organization = self.default_organization,
|
||||
is_global = True,
|
||||
name = data.operating_system.version_major,
|
||||
operating_system = operating_system
|
||||
)
|
||||
|
||||
else: # Create Operating System Version
|
||||
|
||||
operating_system_version = OperatingSystemVersion.objects.create(
|
||||
organization = self.default_organization,
|
||||
is_global = True,
|
||||
name = data.operating_system.version_major,
|
||||
operating_system = operating_system,
|
||||
)
|
||||
|
||||
|
||||
if DeviceOperatingSystem.objects.filter( version=data.operating_system.version, device=device, operating_system_version=operating_system_version ).exists():
|
||||
|
||||
device_operating_system = DeviceOperatingSystem.objects.get(
|
||||
device=device,
|
||||
version = data.operating_system.version,
|
||||
operating_system_version = operating_system_version,
|
||||
)
|
||||
|
||||
if not device_operating_system.installdate: # Only update install date if empty
|
||||
|
||||
device_operating_system.installdate = timezone.now()
|
||||
|
||||
device_operating_system.save()
|
||||
|
||||
else: # Create Operating System Version
|
||||
|
||||
device_operating_system = DeviceOperatingSystem.objects.create(
|
||||
organization = self.default_organization,
|
||||
device=device,
|
||||
version = data.operating_system.version,
|
||||
operating_system_version = operating_system_version,
|
||||
installdate = timezone.now()
|
||||
)
|
||||
|
||||
|
||||
if app_settings.software_is_global:
|
||||
|
||||
software_organization = app_settings.global_organization
|
||||
|
||||
else:
|
||||
|
||||
software_organization = device.organization
|
||||
|
||||
|
||||
if app_settings.software_categories_is_global:
|
||||
|
||||
software_category_organization = app_settings.global_organization
|
||||
|
||||
else:
|
||||
|
||||
software_category_organization = device.organization
|
||||
|
||||
inventoried_software: list = []
|
||||
|
||||
for inventory in list(data.software):
|
||||
|
||||
software = None
|
||||
software_category = None
|
||||
software_version = None
|
||||
|
||||
device_software = None
|
||||
|
||||
software_category = SoftwareCategory.objects.filter( name = inventory.category )
|
||||
|
||||
|
||||
if software_category.exists():
|
||||
|
||||
software_category = SoftwareCategory.objects.get(
|
||||
name = inventory.category
|
||||
)
|
||||
|
||||
else: # Create Software Category
|
||||
|
||||
software_category = SoftwareCategory.objects.create(
|
||||
organization = software_category_organization,
|
||||
is_global = True,
|
||||
name = inventory.category,
|
||||
)
|
||||
|
||||
|
||||
if software_category.name == inventory.category:
|
||||
|
||||
if Software.objects.filter( name = inventory.name ).exists():
|
||||
|
||||
software = Software.objects.get(
|
||||
name = inventory.name
|
||||
)
|
||||
|
||||
if not software.category:
|
||||
|
||||
software.category = software_category
|
||||
software.save()
|
||||
|
||||
else: # Create Software
|
||||
|
||||
software = Software.objects.create(
|
||||
organization = software_organization,
|
||||
is_global = True,
|
||||
name = inventory.name,
|
||||
category = software_category,
|
||||
)
|
||||
|
||||
|
||||
if software.name == inventory.name:
|
||||
|
||||
pattern = r"^(\d+:)?(?P<semver>\d+\.\d+(\.\d+)?)"
|
||||
|
||||
semver = re.search(pattern, str(inventory.version), re.DOTALL)
|
||||
|
||||
|
||||
if semver:
|
||||
|
||||
semver = semver['semver']
|
||||
|
||||
else:
|
||||
semver = inventory.version
|
||||
|
||||
|
||||
if SoftwareVersion.objects.filter( name = semver, software = software ).exists():
|
||||
|
||||
software_version = SoftwareVersion.objects.get(
|
||||
name = semver,
|
||||
software = software,
|
||||
)
|
||||
|
||||
else: # Create Software Category
|
||||
|
||||
software_version = SoftwareVersion.objects.create(
|
||||
organization = self.default_organization,
|
||||
is_global = True,
|
||||
name = semver,
|
||||
software = software,
|
||||
)
|
||||
|
||||
|
||||
if software_version.name == semver:
|
||||
|
||||
if DeviceSoftware.objects.filter( software = software, device=device ).exists():
|
||||
|
||||
device_software = DeviceSoftware.objects.get(
|
||||
device = device,
|
||||
software = software
|
||||
)
|
||||
|
||||
print(f"Select Existing Device Software: {device_software.software.name}")
|
||||
|
||||
else: # Create Software
|
||||
|
||||
device_software = DeviceSoftware.objects.create(
|
||||
organization = self.default_organization,
|
||||
is_global = True,
|
||||
installedversion = software_version,
|
||||
software = software,
|
||||
device = device,
|
||||
action=None
|
||||
)
|
||||
|
||||
|
||||
print(f"Create Device Software: {device_software.software.name}")
|
||||
|
||||
|
||||
if device_software: # Update the Inventoried software
|
||||
|
||||
inventoried_software += [ device_software.id ]
|
||||
|
||||
|
||||
if not device_software.installed: # Only update install date if blank
|
||||
|
||||
device_software.installed = timezone.now()
|
||||
|
||||
device_software.save()
|
||||
|
||||
print(f"Update Device Software (installed): {device_software.software.name}")
|
||||
|
||||
|
||||
if device_software.installedversion.name != software_version.name:
|
||||
|
||||
device_software.installedversion = software_version
|
||||
|
||||
device_software.save()
|
||||
|
||||
print(f"Update Device Software (installedversion): {device_software.software.name}")
|
||||
|
||||
for not_installed in DeviceSoftware.objects.filter( device=device ):
|
||||
|
||||
if not_installed.id not in inventoried_software:
|
||||
|
||||
not_installed.delete()
|
||||
|
||||
print(f"Remove Device Software: {not_installed.software.name}")
|
||||
|
||||
|
||||
if device and operating_system and operating_system_version and device_operating_system:
|
||||
|
||||
|
||||
device.inventorydate = timezone.now()
|
||||
|
||||
device.save()
|
||||
|
||||
if status != Http.Status.CREATED:
|
||||
|
||||
status = Http.Status.OK
|
||||
response_data: dict = {"task_id": f"{task.id}"}
|
||||
|
||||
except PermissionDenied as e:
|
||||
|
||||
|
||||
Reference in New Issue
Block a user