build: bump version 0.3.0 -> 0.4.0

!15

.cz.yaml

@@ -4,5 +4,5 @@ commitizen:
   prerelease_offset: 1
   tag_format: $version
   update_changelog_on_bump: false
-  version: 0.1.2-a1
+  version: 0.4.0
   version_scheme: semver

.gitlab-ci.yml

@@ -19,16 +19,27 @@ variables:
   DOCKER_IMAGE_PUBLISH_REGISTRY: docker.io/nofusscomputing
   DOCKER_IMAGE_PUBLISH_URL: https://hub.docker.com/r/nofusscomputing/$DOCKER_IMAGE_PUBLISH_NAME
 
+  RELEASE_ADDITIONAL_ACTIONS_BUMP: ./.gitlab/additional_actions_bump.sh
+
 
 include:
+
+  - local: .gitlab/unit-test.gitlab-ci.yml
+
   - project: nofusscomputing/projects/gitlab-ci
     ref: development
     file:
       - .gitlab-ci_common.yaml
+      - conventional_commits/.gitlab-ci.yml
       - template/ansible-collection.gitlab-ci.yaml
       - template/mkdocs-documentation.gitlab-ci.yaml
       # ToDo: update gitlabCI jobs for collections workflow
       - git_push_mirror/.gitlab-ci.yml
+      - automation/.gitlab-ci-ansible.yaml
+
+
+Update Git Submodules:
+  extends: .ansible_playbook_git_submodule
 
 
 Github (Push --mirror):
@@ -141,9 +152,9 @@ Github (Push --mirror):
 
       docker buildx imagetools inspect $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG;
 
-      DOCKER_MULTI_ARCH_IMAGES=$(docker buildx imagetools inspect "$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG" --format "{{ range .Manifest.Manifests }}$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG@{{ println .Digest }} {{end}}")
+      #DOCKER_MULTI_ARCH_IMAGES=$(docker buildx imagetools inspect "$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG" --format "{{ range .Manifest.Manifests }}$DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG@{{ println .Digest }} {{end}}")
 
-      echo "[DEBUG] DOCKER_MULTI_ARCH_IMAGES=$DOCKER_MULTI_ARCH_IMAGES";
+      # echo "[DEBUG] DOCKER_MULTI_ARCH_IMAGES=$DOCKER_MULTI_ARCH_IMAGES";
 
       # done;
 
@@ -172,23 +183,23 @@ Github (Push --mirror):
       # fi;
 
 
-      echo "[DEBUG] DOCKER_IMAGE_PUBLISH_NAME=$DOCKER_IMAGE_PUBLISH_NAME";
+      # echo "[DEBUG] DOCKER_IMAGE_PUBLISH_NAME=$DOCKER_IMAGE_PUBLISH_NAME";
 
       # echo "[DEBUG] final: DOCKER_HUB_TAG=$DOCKER_HUB_TAG";
 
-      echo "[DEBUG] DOCKER_MULTI_ARCH_IMAGES=$DOCKER_MULTI_ARCH_IMAGES";
+      # echo "[DEBUG] DOCKER_MULTI_ARCH_IMAGES=$DOCKER_MULTI_ARCH_IMAGES";
-      
+
-      
+
 
       if [ "${CI_COMMIT_TAG}" ]; then
 
-        docker buildx imagetools create $DOCKER_MULTI_ARCH_IMAGES --tag $DOCKER_IMAGE_PUBLISH_REGISTRY/$DOCKER_IMAGE_PUBLISH_NAME:$CI_COMMIT_TAG;
+        docker buildx imagetools create $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG --tag $DOCKER_IMAGE_PUBLISH_REGISTRY/$DOCKER_IMAGE_PUBLISH_NAME:$CI_COMMIT_TAG;
 
-        docker buildx imagetools create $DOCKER_MULTI_ARCH_IMAGES --tag $DOCKER_IMAGE_PUBLISH_REGISTRY/$DOCKER_IMAGE_PUBLISH_NAME:latest;
+        docker buildx imagetools create $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG --tag $DOCKER_IMAGE_PUBLISH_REGISTRY/$DOCKER_IMAGE_PUBLISH_NAME:latest;
 
       else
 
-        docker buildx imagetools create $DOCKER_MULTI_ARCH_IMAGES --tag $DOCKER_IMAGE_PUBLISH_REGISTRY/$DOCKER_IMAGE_PUBLISH_NAME:dev;
+        docker buildx imagetools create $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG --tag $DOCKER_IMAGE_PUBLISH_REGISTRY/$DOCKER_IMAGE_PUBLISH_NAME:dev;
 
       fi;
 
@@ -234,10 +245,31 @@ Docker Container (dev):
     - when: never
 
 
+test:unit:docker:collection:installed:dev:
+  extends: .docker_collection_installed
+  needs: 
+    - "Docker Container (dev)"
+  rules:
+    - if: $CI_COMMIT_TAG
+      when: never
+
+    - if: "$CI_COMMIT_AUTHOR =='nfc_bot <helpdesk@nofusscomputing.com>'"
+      when: never
+
+    - if:
+        $CI_COMMIT_BRANCH != "master"
+          &&
+        $CI_PIPELINE_SOURCE == "push"
+      when: always
+
+    - when: never
+
+
 Docker Hub (dev):
   extends: .publish-docker-hub
   needs: 
     - "Docker Container (dev)"
+    - "test:unit:docker:collection:installed:dev"
   resource_group: docker-build
   rules:
     - if: $CI_COMMIT_TAG
@@ -250,7 +282,7 @@ Docker Hub (dev):
         $CI_COMMIT_BRANCH == "development"
           &&
         $CI_PIPELINE_SOURCE == "push"
-      when: always
+      when: on_success
 
     - when: never
 
@@ -276,10 +308,31 @@ Docker Container:
     - when: never
 
 
+test:unit:docker:collection:installed:
+  extends: .docker_collection_installed
+  needs:
+    - Docker Container
+  rules:
+    - if: $CI_COMMIT_TAG
+      when: on_success
+
+    - if: "$CI_COMMIT_AUTHOR =='nfc_bot <helpdesk@nofusscomputing.com>'"
+      when: never
+
+    # - if: # Occur on merge
+    #     $CI_COMMIT_BRANCH == "development"
+    #       &&
+    #     $CI_PIPELINE_SOURCE == "push"
+    #   when: always
+
+    - when: never
+
+
 Docker Hub:
   extends: .publish-docker-hub
   needs: 
     - "Docker Container"
+    - "test:unit:docker:collection:installed"
     - "Gitlab Release"
   resource_group: docker-build
   rules:

.gitlab/additional_actions_bump.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+sed -E "/http_agent: nfc-phpipam-scan-agent/s/\/(.+)/\/$(cz version --project)/g" -i playbooks/tasks/scan_subnet.yaml;
+
+git add playbooks/tasks/scan_subnet.yaml;
+
+git status;
+
+sed -E "/http_agent: nfc-phpipam-scan-agent/s/\/(.+)/\/$(cz version --project)/g" -i playbooks/tasks/api_call.yaml;
+
+git add playbooks/tasks/api_call.yaml;
+
+git status;

.gitlab/unit-test.gitlab-ci.yml

@@ -0,0 +1,33 @@
+
+.docker_collection_installed:
+  stage: test
+  image: 
+    name: nofusscomputing/docker-buildx-qemu:dev
+    pull_policy: always
+  services:
+    - name: docker:23-dind
+      entrypoint: ["env", "-u", "DOCKER_HOST"]
+      command: ["dockerd-entrypoint.sh"]
+  variables:
+    DOCKER_HOST: tcp://docker:2375/
+    DOCKER_DRIVER: overlay2
+    DOCKER_DOCKERFILE: dockerfile
+    # See https://github.com/docker-library/docker/pull/166
+    DOCKER_TLS_CERTDIR: ""
+    GIT_STRATEGY: none
+
+  before_script:
+
+    - | # Pull the image
+      docker pull $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG;
+
+    - | # Show local images
+      docker images;
+
+  script:
+
+    - | # Confirm the package is installed within the docker container
+      docker run \
+        --rm \
+        $DOCKER_IMAGE_BUILD_REGISTRY/$DOCKER_IMAGE_BUILD_NAME:$DOCKER_IMAGE_BUILD_TAG \
+        ansible-galaxy collection list | grep nofusscomputing.phpipam_scan_agent;

CHANGELOG.md

@@ -1,9 +1,53 @@
-## Unreleased
+## 0.4.0 (2024-02-26)
+
+### Feat
+
+- **docker**: update trusted CA certificates on start
+- **docker**: add trace output to determine install path for package
+
+## 0.3.0 (2024-02-25)
+
+### Feat
+
+- **server**: DNS Reverse Lookup for IP Address'
+- **server**: Convert scanned time within scan report to UTC
 
 ### Fix
 
+- **ci**: http user_agent version set during version bump
+- **server**: Time of last access for scan agent to use 'now'
+
+## 0.2.0 (2024-02-24)
+
+### Feat
+
+- **server**: Update the time of last access for scan agent
+- **scanner**: Add scanner timezone to scan report
+- **scanner**: don't include PTR DNS records as hostname
+- **scanner**: for ssl cert validation when uploading to server
+- **scanner**: if an auth token has been set, fail non-https communication with server
+- **scanner**: Set user http user agent to <name>/<version>
+- **server**: scanner identity confirmation
+- **server**: update subnet discovery date/time
+- **server**: move config of variables to vars file
+- **agent**: move config of variables to vars file
+- **server**: if hostname in scan report, update the database
+- **agent**: if hostname present in nmap scan report, add to report for server
+- **agent**: showsubnet address in logs when conducting subnet actions
+- **server**: show ip address in logs when updating an ip address
+- **api_call**: before returning check if cached file exists
+
+### Fix
+
+- **server**: use correct sql syntax to insert ipaddress
+- **agent**: only attempt to scan subnet if subnets were returned
 - **build**: ensure the correct tags and architectures are published
 
+### Refactor
+
+- **server**: don't process scan results in parallel
+- **api_call**: use var name that makes more sense
+
 ## 0.1.1 (2024-02-21)
 
 ### Fix
@@ -15,12 +59,9 @@
 ### Feat
 
 - **docker**: adjust scanner job to run every 5 mins
-- **docs**: add initial doc layout and CI jobs
 
 ### Fix
 
 - **ci**: ensure docker publish uses correct tag
 - **ci**: ensure correct github sync repo is used
 - **ci**: ensure docker build and publish occurs
-- Don't process results if scan report is empty
-- **docs**: correct nav link

README.md

@@ -1,9 +1,13 @@
-<span align="center">
+<span style="text-align: center;">
 
 # No Fuss Computing - Ansible Collection PHPIPAM Scan Agent
 
 ![Project Status - Active](https://img.shields.io/badge/Project%20Status-Active-green?logo=gitlab&style=plastic) 
 
+[![Downloads](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgalaxy.ansible.com%2Fapi%2Fv3%2Fplugin%2Fansible%2Fcontent%2Fpublished%2Fcollections%2Findex%2Fnofusscomputing%2Fphpipam_scan_agent%2F&query=%24.download_count&style=plastic&logo=ansible&logoColor=white&label=Galaxy%20Downloads&labelColor=black&color=cyan)](https://galaxy.ansible.com/ui/repo/published/nofusscomputing/phpipam_scan_agent/)
+[![Docker Pulls](https://img.shields.io/docker/pulls/nofusscomputing/phpipam-scan-agent?style=plastic&logo=docker&logoColor=0db7ed&color=0db7ed)](https://hub.docker.com/r/nofusscomputing/phpipam-scan-agent)
+
+
 ----
 
 <br>
@@ -28,20 +32,6 @@ This project is hosted on [gitlab](https://gitlab.com/nofusscomputing/projects/a
 
 ![Gitlab build status - development](https://img.shields.io/badge/dynamic/json?color=ff782e&label=Build&query=0.status&url=https%3A%2F%2Fgitlab.com%2Fapi%2Fv4%2Fprojects%2F55052132%2Fpipelines%3Fref%3Ddevelopment&logo=gitlab&style=plastic) ![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/collections/phpipam_scan_agent%2F-%2Fraw%2Fdevelopment%2F.cz.yaml)
 
-----
-
-**Ansible Galaxy**
-
-[![Latest Version](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgalaxy.ansible.com%2Fapi%2Fv3%2Fplugin%2Fansible%2Fcontent%2Fpublished%2Fcollections%2Findex%2Fnofusscomputing%2Fphpipam_scan_agent%2F&query=%24.highest_version.version&style=plastic&logo=ansible&logoColor=white&label=Latest%20Release&labelColor=black&color=cyan)](https://galaxy.ansible.com/ui/repo/published/nofusscomputing/phpipam_scan_agent/)
-![Downloads](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgalaxy.ansible.com%2Fapi%2Fv3%2Fplugin%2Fansible%2Fcontent%2Fpublished%2Fcollections%2Findex%2Fnofusscomputing%2Fphpipam_scan_agent%2F&query=%24.download_count&style=plastic&logo=ansible&logoColor=white&label=Downloads&labelColor=black&color=cyan)
-
-----
-
-**Docker Hub**
-
-[![Docker Image Version](https://img.shields.io/docker/v/nofusscomputing/phpipam-scan-agent?sort=semver&style=plastic&logo=docker&logoColor=0db7ed&color=0db7ed&label=Latest%20Release)](https://hub.docker.com/r/nofusscomputing/phpipam-scan-agent)
-[![Docker Pulls](https://img.shields.io/docker/pulls/nofusscomputing/phpipam-scan-agent?style=plastic&logo=docker&logoColor=0db7ed&color=0db7ed)](https://hub.docker.com/r/nofusscomputing/phpipam-scan-agent)
-
 </span>
 
 ----

dockerfile

@@ -92,21 +92,26 @@ COPY includes/ /
 
 RUN mkdir -p /tmp/collection; \
   if [ "$COLLECTION_PACKAGE" != "dev" ]; then \
-    echo "specified"; \
+    echo "[TRACE] Package Specified"; \
     ansible-galaxy collection install --force-with-deps --pre \
-      $COLLECTION; \
+      $COLLECTION_PACKAGE; \
   elif [ "$COLLECTION_PACKAGE" == "dev" ]; then \
+    echo "[TRACE] Development Build"; \
     git clone \
       --depth=1 \
       -b $COLLECTION_BRANCH \ 
       https://gitlab.com/nofusscomputing/projects/ansible/collections/phpipam_scan_agent.git \
       /tmp/collection; \
     if [ "${COLLECTION_COMMIT}" != "none" ]; then git switch $COLLECTION_COMMIT; fi; \
+    echo "[TRACE] Installing Development Build"; \
     ansible-galaxy collection install --force-with-deps --pre \
     /tmp/collection/.; \
     rm -Rf /tmp/collection; \
   fi; \
-  chmod +x /etc/cron.d/*;
+  chmod +x /etc/cron.d/*; \
+  chmod +x /bin/update-ca; \
+  chown root:root -R /etc/phpipam; \
+  chmod 740 -R /etc/phpipam;
 
 
 WORKDIR /root

docs/projects/ansible/collection/phpipam_scan_agent/docker.md

@@ -19,16 +19,12 @@ Launching the docker container can be done with
 
 docker run \
     -d \
-    -e "API_URL=<your value here>" \
-    -e "MYSQL_HOST=<your value here>" \
-    -e "MYSQL_USER=<your value here>" \
-    -e "MYSQL_PASSWORD=<your value here>" \
-    -e "SCANNER_TOKEN=<your value here>" \
-    -e "SCANNER_NAME=<your value here>" \
-    -e "SCANNER_CODE=<your value here>" \
     -p "5000:5000" \
+    -v "./scan_agent.yaml:/etc/phpipam/scan_agent.yaml" \
+    -v "./scan_server.yaml:/etc/phpipam/scan_server.yaml" \
+    -v "./my-custom-ca.crt:/usr/local/share/ca-certificates/my-custom-ca.crt:ro"
     --name scan-agent \
-    scan-agent:latest;
+    nofusscomputing/phpipam-scan-agent:latest;
 
 ```
 
@@ -42,9 +38,20 @@ When viewing the container logs `docker logs <container name>`, what you will se
 During the build of the container environmental variable `ANSIBLE_FORCE_COLOR='true'` is set, this enables the playbooks to be in colour when viewing the container logs. If this is not desired, set the variable to `ANSIBLE_FORCE_COLOR='false'` when launching the container.
 
 
+### Custom CA Certificate
+
+When the container starts or restarts a script runs that updates the containers trusted CA certificates. Details for mounting a/an certificate(s) is detailed below.
+
+
 ### Volumes
 
-There are no volumes for this container.
+To configure the components, the following files will need to be mounted within the container:
+
+- Custom CA certificate at path `/usr/local/share/ca-certificates/`. you can mount a single `crt` file or multiple if you use a sub-directory at this path.
+
+- scanner config file at path `/etc/phpipam/scan_agent.yaml`, see [scanner docs](scanner.md#variables) for details.
+
+- server config at path `/etc/phpipam/scan_server.yaml`, see [server docs](server.md#variables) for details.
 
 If you wish to customize the cronjob for the scan component within the container, mount a new cron file to path `/etc/cron.d/scanner`. The default cron file is as follows:
 

docs/projects/ansible/collection/phpipam_scan_agent/index.md

@@ -6,14 +6,16 @@ template: project.html
 about: https://gitlab.com/nofusscomputing/projects/ansible/collections/phpipam_scan_agent
 ---
 
-<span align="center">
+<span style="text-align: center;">
 
 ![Project Status - Active](https://img.shields.io/badge/Project%20Status-Active-green?logo=gitlab&style=plastic)
 
-[![Latest Version](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgalaxy.ansible.com%2Fapi%2Fv3%2Fplugin%2Fansible%2Fcontent%2Fpublished%2Fcollections%2Findex%2Fnofusscomputing%2Fphp_scan_agent%2F&query=%24.highest_version.version&style=plastic&logo=ansible&logoColor=white&label=Latest%20Release&labelColor=black&color=cyan)](https://galaxy.ansible.com/ui/repo/published/nofusscomputing/ci_test_collection/)
-![Downloads](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgalaxy.ansible.com%2Fapi%2Fv3%2Fplugin%2Fansible%2Fcontent%2Fpublished%2Fcollections%2Findex%2Fnofusscomputing%2Fphp_scan_agent%2F&query=%24.download_count&style=plastic&logo=ansible&logoColor=white&label=Downloads&labelColor=black&color=cyan)
 
-[![Docker Image Version](https://img.shields.io/docker/v/nofusscomputing/phpipam-scan-agent?sort=semver&style=plastic&logo=docker&logoColor=0db7ed&color=0db7ed&label=Latest%20Release)](https://hub.docker.com/r/nofusscomputing/phpipam-scan-agent)
+![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Stable%20Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/collections/phpipam_scan_agent%2F-%2Fraw%2Fmaster%2F.cz.yaml) 
+![branch release version](https://img.shields.io/badge/dynamic/yaml?color=ff782e&logo=gitlab&style=plastic&label=Dev%20Release&query=%24.commitizen.version&url=https%3A//gitlab.com/nofusscomputing/projects/ansible/collections/phpipam_scan_agent%2F-%2Fraw%2Fdevelopment%2F.cz.yaml)
+
+
+[![Downloads](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fgalaxy.ansible.com%2Fapi%2Fv3%2Fplugin%2Fansible%2Fcontent%2Fpublished%2Fcollections%2Findex%2Fnofusscomputing%2Fphpipam_scan_agent%2F&query=%24.download_count&style=plastic&logo=ansible&logoColor=white&label=Galaxy%20Downloads&labelColor=black&color=cyan)](https://galaxy.ansible.com/ui/repo/published/nofusscomputing/phpipam_scan_agent/)
 [![Docker Pulls](https://img.shields.io/docker/pulls/nofusscomputing/phpipam-scan-agent?style=plastic&logo=docker&logoColor=0db7ed&color=0db7ed)](https://hub.docker.com/r/nofusscomputing/phpipam-scan-agent)
 
 
@@ -23,6 +25,8 @@ A phpIPAM scan agent designed for both local and remote network scanning. This A
 
 This collection has been broken down into two components, a [server](server.md) and a [scanner](scanner.md). The scanner as the name implies will scan the networks assigned to it by phpIPAM and on completing a scan of a subnet, will post the results to the Server component which will process the results, and update the phpIPAM MySQL/MariaDB database directly.
 
+This agent has been created to solve a few shortcomings of the official scan-agent. Ansible was chosen as the tool for the creation of the scan-agent. This was done as it lowers the bar of entry to modifying and development. This as well as that Ansible is a mature automation system that has many modules. The latter offers a wide range of integrating different systems. The scanner component is just an ansible playbook, whilst the server is an ansilbe rulebook which is based upon Event Driven Ansible (EDA).
+
 
 ## Installation
 
@@ -30,6 +34,8 @@ This collection is available on Ansible Galaxy and can be installed with `ansibl
 
 Prefer to use our [docker](docker.md) image? It's available on Docker Hub `docker pull nofusscomputing/phpipam-scan-agent:latest`.
 
+There is no timezone support within phpIPAM. Due to this it's recommended that you setup phpIPAM, the database and the scanner component to use UTC (GMT +00:00) timezone or that all components be setup to use the same timezone.
+
 
 ## Features
 
@@ -48,9 +54,11 @@ The following features are available or planned to be implmented:
     !!! info
         It's only possible to obtain a MAC Address if the scanner is on the same L2 network (Broadcast Domain). Within the docs you will find the different methods available to achieve this.
 
-- [**ToDo** Remote Network Scanning](https://gitlab.com/nofusscomputing/projects/ansible/collections/phpipam_scan_agent/-/issues/1)
+- [Remote Network Scanning](scanner.md#remote-network-scannning)
 
-- [**ToDo** Resolve DNS names](https://gitlab.com/nofusscomputing/projects/ansible/collections/phpipam_scan_agent/-/issues/4)
+- [Resolve DNS names](index.md#resolve-dns-names)
+
+- [Timezone Normalization](server.md#timezone-normalization)
 
 
 ## phpIPAM Features
@@ -68,6 +76,32 @@ When this setting is enabled, the scanner will scan the entire subnet that has b
     the work in [Hosts check](https://gitlab.com/nofusscomputing/projects/ansible/collections/phpipam_scan_agent/-/issues/3), will adjust this behaviour to only add hosts that dont exist.
 
 
+ - Location `Subnet -> Discover new hosts [Last discovery time]` 
+
+This is displayed within the subnet interface next to the `Discover new hosts` field. Whenever a scan report is received by the server this filed is updated to reflect the last scan time.
+
+
+### Resolve DNS names
+
+- Location `Subnet -> Resolve DNS names`
+
+When this setting is enabled, DNS resolution for the hostname for any IP found will be conducted. If you have setup nameservers (`Administration -> Nameservers`) and assigned them to the subnet, they will be used for name resolution. If no nameservers are configured for the subnet, the host's nameservers where the scanner is running from will be used. BY default within phpIPAM, DNS Resolution is turned off for a subnet and as such the scanner will not resolve IP address DNS names.
+
+!!! info
+    The FQDN for an IP that is retured is not used as the hostname. **Only** the first octet. i.e. if the reverse DNS lookup returns a FQDN of `my_hostname.domainname.tld`, the hostname for the IP address will be set to `my_hostname`
+
+
+### Scan Agent
+
+- Location `Administration -> Scan agents [Last access]`
+
+This is displayed within the Scan Agent table in column `Last access`. When ever the scanner checks in, the time of the checkin is used to update the field.
+
+- Location `Subnet -> Scan agent [Last Check]`
+
+This is displayed within the subnet interface next to `Scan agent` field. When ever the scanner checks in, the time of the checkin is used to update the field.
+
+
 ## Development Notes
 
 Contributions to this project are welcome. Below you will find some useful commands for use during development.
@@ -80,13 +114,8 @@ docker build . --tag scan-agent:dev --build-arg  COLLECTION_BRANCH=<your feature
 # Launch your build container
 docker run \
     -d \
-    -e "API_URL=<your value here>" \
+    -v "./scan_agent.yaml:/etc/phpipam/scan_agent.yaml" \
-    -e "MYSQL_HOST=<your value here>" \
+    -v "./scan_server.yaml:/etc/phpipam/scan_server.yaml" \
-    -e "MYSQL_USER=<your value here>" \
-    -e "MYSQL_PASSWORD=<your value here>" \
-    -e "SCANNER_TOKEN=<your value here>" \
-    -e "SCANNER_NAME=<your value here>" \
-    -e "SCANNER_CODE=<your value here>" \
     -e "ANSIBLE_LOG_PATH=/var/log/ansible.log" \
     -p "5000:5000" \
     --name scan-agent \

docs/projects/ansible/collection/phpipam_scan_agent/scanner.md

@@ -6,20 +6,16 @@ template: project.html
 about: https://gitlab.com/nofusscomputing/projects/ansible/collections/phpipam_scan_agent
 ---
 
-The Scan Agent Scanner component is intended to scan networks that are assigned to it by the phpIPAM server. It can be installed and ran from any host that is capable of running python.
+The Scan Agent Scanner component is intended to scan networks that are assigned to it by the phpIPAM server. It can be installed and ran from any host that is capable of running python. The scan agent only requires that there be `nmap` installed as this is the package that conducts the network scanning.
 
 
 ## Usage
 
-After installing the collection, running the agent is as simple as running the following command:
+Ensure that the `namp` package is installed, install and configure the collection. Running the agent is as simple as running the following command:
 
 ``` bash
 
-ansible-playbook nofusscomputing.phpipam_scan_agent.agent \
+ansible-playbook nofusscomputing.phpipam_scan_agent.agent
-  --extra-vars "api_url=<your value here>" \
-  --extra-vars "client_token=<your value here>" \
-  --extra-vars "client_name=<your value here>" \
-  --extra-vars "scanagent_code=<your value here>"
 
 ```
 
@@ -33,24 +29,25 @@ The scanner component requires API access to phpIPAM. THe API user that is used,
 
 ### Variables
 
-The variables described below, if optional the value specified here is the default value. Any variable that can be set via environmental variables have the variable name enclosed in `[]`
+Variables for the scanner are set in a variables file at path `/etc/phpipam/scan_agent.yaml`. The variables described below, if optional the value specified here is the default value.
 
 ``` yaml
+nofusscomputing_phpipam_scan_agent:
 
-client_token: ""                       # Mandatory, String client api token to connect to phpIPAM API [SCANNER_TOKEN]
+  api_url:                           # Mandatory, String. url with protocol of the phpIPAM API to connect to.
-client_name: ""                        # Mandatory, String. The scanner name as set in phpIPAM interface [SCANNER_NAME]
-scanagent_code: ""                     # Mandatory, String. Scan Agent Code as set in phpIPAM interface [SCANNER_CODE]
 
+  http_port: 5000                    # Optional, Integer. http port to connect to the server.
+  http_server: http://127.0.0.1      # Optional, Integer. url with protocol of the Scan Server to connect to.
+  auth_token:                        # Optional, String. The Scan-Agent server authentication token.
 
-nfc_c_http_port: 5000                  # Optional, Integer. http port to connect to the server. [HTTP_PORT]
+  cache_expire_time: 1800            # Optional, Integer. Time in seconds to expire the phpIPAM cache.
-nfc_c_http_server: http://127.0.0.1    # Optional, Integer. url with protocol of the Scan Server to connect to. [HTTP_URL]
+  epoch_time_offset: 0               # optional, int. Value in seconds to offset the time
 
+# phpIPAM Scan Agent Settings
+  client_token:                      # Mandatory, String client api token to connect to phpIPAM API
+  client_name:                       # Mandatory, String. The scanner name as set in phpIPAM interface
+  scanagent_code:                    # Mandatory, String. Scan Agent Code as set in phpIPAM interface
 
-api_url: http://127.0.0.1              # Optional, String. url with protocol of the phpIPAM API to connect to. [API_URL]
-
-
-nfc_c_cache_expire_time: 1800          # Optional, Integer. Time in seconds to expire the phpIPAM cache.
-nfc_c_epoch_time_offset: 0             # optional, int. Value in seconds to offset the time
 
 ```
 
@@ -63,13 +60,11 @@ nfc_c_epoch_time_offset: 0             # optional, int. Value in seconds to offs
 These images are of the phpIPAM interface that show in green text the variable name that would be set as detailed above.
 
 ![phpIPAM API](images/phpipam_api.png)
-
 phpIPAM API Settings
 
 ----
 
 ![phpIPAM Scan Agent](images/phpipam_scan_agent_details.png)
-
 phpIPAM Scan Agent Settings
 
 
@@ -99,3 +94,15 @@ The scanner component has the following workflow:
     1. upload scan report to configured Server.
 
 1. workflow complete.
+
+
+## Remote network Scannning
+
+Once the [server component](server.md#remote-network-scannning) has been setup, the client can be installed/used from any network. Even a network that is isolated from the server. Only caveat is that the client can communicate with the server. To ensure that the client can connect to the server set the `auth_token` to match that of the server.
+
+There is no true confirmation of the servers identity outside of confirming the TLS Certificate is trusted. Due to this fact, you're advised to use your own CA to sign the server components TLS Certificate. By doing this only you can issue a certificate to the server component. All that is required is to ensure that [your CA certificate](docker.md#custom-ca-certificate) is within the [trusted certificates](../../../itil/runbooks/linux/custom_ca.md) of the machine that is running the agent.
+
+!!! danger "Security"
+    Failing to secure the server component communication with TLS will allow anyone with direct access to the line of communication to view the `auth_token`. Anyone who has the `auth_token` will be able to upload data to the server.
+
+    In an attempt to mitigate this, the scanner will fail to communicate with the server if you have set an `auth_token` and attempt non-TLS communication with the server.

docs/projects/ansible/collection/phpipam_scan_agent/server.md

@@ -22,24 +22,27 @@ ansible-rulebook -r nofusscomputing.phpipam_scan_agent.agent_receive
 
 ### Variables
 
-The variables described below, if optional the value specified here is the default value. All variables that are used by the server component are environmental variables that must be set before execution.
+The variables described below, if optional the value specified here is the default value. The variables are to be set in a variables file at path `/etc/phpipam/scan_server.yaml`
 
-``` bash
+``` yaml
 
-# phpIPAM MariaDB/MySQL Variables
+nofusscomputing_phpipam_scan_server:
-MYSQL_HOST=            # Mandatory, String. IP/DNS of host to connect.
+
-MYSQL_PORT=3306        # Optional, Integer. port to use for connection.
+  # phpIPAM MariaDB/MySQL Variables
-MYSQL_USER=            # Mandatory, String. User to authenticate with.
+  mysql_host:         # Mandatory, String. IP/DNS of host to connect.
-MYSQL_PASSWORD=        # Mandatory, String. Password for the user to connect with.
+  mysql_port: 3306    # Optional, Integer. port to use for connection.
+  mysql_user:         # Mandatory, String. User to authenticate with.
+  mysql_password:     # Mandatory, String. Password for the user to connect with.
 
 
-# Server Component Variables
+  # Server Component Variables
-HTTP_PORT=5000         # Optional, Integer. The port for the Server component to listen for connections.
+  http_port: 5000     # Optional, Integer. The port for the Server component to listen for connections.
+  auth_token:         # Optional, String. Token used to authentication Agents.
 
 ```
 
 
-# Workflow
+## Workflow
 
 The Server componet has the following workflow:
 
@@ -54,3 +57,17 @@ The Server componet has the following workflow:
     - _if no results found, no further processing occurs_
 
 1. Update the phpIPAM MariaDB/MySQL database directly
+
+
+## Remote network Scannning
+
+Remote network scanning is possible with the Scan-Agent. The server must be setup and have connectivity to the phpIPAM MariaDB/MySQL database. Currently the server does not perform secure communication. As such you are strongly encouraged to setup the server component behind a reverse proxy that conducts the TLS termination. It's also recommended that you use your [own CA](docker.md#custom-ca-certificate) to sign the TLS certificate. Just don't forget to add this CA certificate to the [trusted CA certificates](../../../itil/runbooks/linux/custom_ca.md)
+
+The [scan](scanner.md#remote-network-scannning) and server component must be setup with the same `auth_token`. It is this token that provides a means to ensure that what the server is receiving, is from an authorized client.
+
+!!! danger "Security"
+    Failing to secure the server component communication with TLS will allow anyone with direct access to the line of communication to view the `auth_token`. Anyone who has the `auth_token` will be able to upload data to the server.
+
+## Timezone normalization
+
+As there is no support within phpIPAM for different timezones. The server component when receiving updates from scanners, will convert any time found to UTC (GMT +00:00). This is required so that phpIPAM features that rely on time, function as they should. This setup requires that machine or docker containers for phpIPAM and the MySQL/MariaDB database both have their timezones set to UTC. If you don't wish for any timezone conversion to be done, ensure that where ever all components, including the scan server component, share the same timezone.

docs/projects/itil/runbooks/linux/custom_ca.md

@@ -0,0 +1 @@
+linked page

extensions/eda/rulebooks/agent_receive.yml

@@ -1,12 +1,12 @@
 - name: Agent Webhook
   hosts: all
-  execution_strategy: parallel
 
   sources:
     - name: Webhook
       ansible.eda.webhook:
         host: 0.0.0.0
-        port: "{{ HTTP_PORT }}"
+        port: "{{ nofusscomputing_phpipam_scan_server.http_port | default(5000) | int }}"
+        token: "{{ nofusscomputing_phpipam_scan_server.auth_token | default('no-token-set') }}"
 
   rules:
 

galaxy.yml

@@ -8,7 +8,7 @@ namespace: nofusscomputing
 name: phpipam_scan_agent
 
 # The version of the collection. Must be compatible with semantic versioning
-version: 0.1.2-a1
+version: 0.4.0
 
 # The path to the Markdown (.md) readme file. This path is relative to the root of the collection
 readme: README.md

includes/bin/update-ca

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+set -e;
+
+run_command() {
+
+  command_date=$(date '+%Y-%m-%d %H:%M:%S');
+
+  echo "${command_date} - info - $1 - $2";
+
+  $1
+
+  if [ $? ]; then
+
+    echo "${command_date} - info - $1 - command ran without error. exit=$?";
+
+  else
+
+    echo "${command_date} - error - ${1} - command had an error. exit=$?";
+
+  fi
+
+}
+
+
+while :
+do
+
+  run_command "ls -lR /usr/local/share/ca-certificates/" "List custom CA Certificates";
+
+
+  run_command "update-ca-certificates --fresh" "Clear currently trusted CA Certificates";
+
+
+  run_command "update-ca-certificates" "Update trusted CA Certificates";
+
+
+  sleep 9999999999999;
+
+done;

includes/etc/phpipam/scan_agent.yaml

@@ -0,0 +1,20 @@
+---
+
+nofusscomputing_phpipam_scan_agent:
+
+  # api_url: http://127.0.0.1          # Mandatory, String. url with protocol of the phpIPAM API to connect to.
+
+  # http_port: 5000                    # Optional, Integer. http port to connect to the server.
+  # http_server: http://127.0.0.1      # Optional, Integer. url with protocol of the Scan Server to connect to.
+
+
+  # cache_expire_time: 1800            # Optional, Integer. Time in seconds to expire the phpIPAM cache.
+  # epoch_time_offset: 0               # optional, int. Value in seconds to offset the time
+
+
+# phpIPAM Scan Agent Settings
+  client_token:                      # Mandatory, String client api token to connect to phpIPAM API [client_token]
+  client_name:                       # Mandatory, String. The scanner name as set in phpIPAM interface [client_name]
+  scanagent_code:                    # Mandatory, String. Scan Agent Code as set in phpIPAM interface [scanagent_code]
+
+  # auth_token:                        # Optional, String. The Scan-Agent server authentication token.

includes/etc/phpipam/scan_server.yaml

@@ -0,0 +1,15 @@
+---
+
+nofusscomputing_phpipam_scan_server:
+
+# phpIPAM MariaDB/MySQL Variables
+# mysql_host:         # Mandatory, String. IP/DNS of host to connect. [nfc_c_]
+# mysql_port: 3306    # Optional, Integer. port to use for connection. [nfc_c_]
+# mysql_user:         # Mandatory, String. User to authenticate with. [nfc_c_]
+# mysql_password:     # Mandatory, String. Password for the user to connect with. [nfc_c_]
+
+
+# Server Component Variables
+# http_port: 5000     # Optional, Integer. The port for the Server component to listen for connections.
+
+# auth_token:         # Optional, String. Token used to authentication Agents.

includes/etc/supervisor/conf.d/rulebook.conf

@@ -1,7 +1,7 @@
 [program:rulebook]
 startsecs=0
 stopwaitsecs=55
-command=ansible-rulebook -r nofusscomputing.phpipam_scan_agent.agent_receive --env-vars "HTTP_PORT" -i /root/hosts.yaml -v
+command=ansible-rulebook -r nofusscomputing.phpipam_scan_agent.agent_receive --vars /etc/phpipam/scan_server.yaml -i /root/hosts.yaml -v
 autorestart=true
 autostart=true
 stdout_logfile=/dev/fd/1

includes/etc/supervisor/conf.d/update-ca-certificates.conf

@@ -0,0 +1,8 @@
+[program:trusted-ca-update]
+startsecs=0
+stopwaitsecs=55
+command=/bin/update-ca
+autorestart=true
+autostart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0

playbooks/agent.yaml

@@ -5,64 +5,48 @@
 
   tasks:
 
+    - name: Check for Existance of config file
+      ansible.builtin.stat:
+        path: /etc/phpipam/scan_agent.yaml
+      register: config_file_check
 
-    - name: Fetch Required Environmental Variables
+
-      ansible.builtin.set_fact:
+    - name: Load Config File
-        client_token: "{{ lookup('ansible.builtin.env', 'SCANNER_TOKEN') | default('') }}"
+      ansible.builtin.include_vars:
-        client_name: "{{ lookup('ansible.builtin.env', 'SCANNER_NAME') }}"
+        file: /etc/phpipam/scan_agent.yaml
-        scanagent_code: "{{ lookup('ansible.builtin.env', 'SCANNER_CODE') | default('') }}"
-        api_url: "{{ lookup('ansible.builtin.env', 'API_URL') | default('') }}"
-      no_log: true
       when: >
-        client_token is not defined
+        config_file_check.stat.exists
-          and
-        client_name is not defined
-          and
-        scanagent_code is not defined
-          and
-        api_url is not defined
-
-
-    - name: Fetch Required Environmental Variable - HTTP_URL
-      ansible.builtin.set_fact:
-        nfc_c_http_server: "{{ lookup('ansible.builtin.env', 'HTTP_URL') | default('') }}"
-      when: >
-        lookup('ansible.builtin.env', 'HTTP_URL') | default('') != ''
-
-
-    - name: Fetch Required Environmental Variable - HTTP_PORT
-      ansible.builtin.set_fact:
-        nfc_c_http_port: "{{ lookup('ansible.builtin.env', 'HTTP_PORT') | default('') }}"
-      when: >
-        lookup('ansible.builtin.env', 'HTTP_PORT') | default('') != ''
 
 
     - name: Confirm 'api_url' is Set
       ansible.builtin.assert:
         that:
-          - api_url is defined
+          - nofusscomputing_phpipam_scan_agent.api_url is defined
-          - api_url != ''
+          - nofusscomputing_phpipam_scan_agent.api_url != ''
         msg: "missing Required Variables"
 
 
     - name: Confirm 'client_token' is Set
       ansible.builtin.assert:
         that:
-          - client_token is defined
+          - nofusscomputing_phpipam_scan_agent.client_token is defined
+          - nofusscomputing_phpipam_scan_agent.client_token != ''
         msg: "missing Required Variables"
 
 
     - name: Confirm 'client_name' is Set
       ansible.builtin.assert:
         that:
-          - client_name is defined
+          - nofusscomputing_phpipam_scan_agent.client_name is defined
+          - nofusscomputing_phpipam_scan_agent.client_name != ''
         msg: "missing Required Variables"
 
 
     - name: Confirm 'scanagent_code' is Set
       ansible.builtin.assert:
         that:
-          - scanagent_code is defined
+          - nofusscomputing_phpipam_scan_agent.scanagent_code is defined
+          - nofusscomputing_phpipam_scan_agent.scanagent_code != ''
         msg: "missing Required Variables"
 
 
@@ -84,17 +68,18 @@
     - name: Scan Subnet
       ansible.builtin.include_tasks:
         file: tasks/scan_subnet.yaml
-      loop: "{{ nfc_c_scan_agent_subnets }}"
+      loop: "{{ nfc_c_scan_agent_subnets | default([]) }}"
       loop_control:
         loop_var: subnet
 
 
-  vars: # ToDo: remove the below t4est vars
+  vars:
+    nfc_c_cache_expire_time: 1800
     nfc_c_http_port: 5000
     nfc_c_http_server: http://127.0.0.1
+    nfc_c_path_cache: "{{ playbook_dir }}/../cache"
 
     api_address: addresses
     api_subnets: subnets
     api_scanagents: tools/scanagents
-    nfc_c_path_cache: "{{ playbook_dir }}/../cache"
+    api_nameservers: tools/nameservers
-    nfc_c_cache_expire_time: 1800

playbooks/server.yaml

@@ -7,26 +7,29 @@
   tasks:
 
 
-    - name: Fetch Required Environmental Variables
-      ansible.builtin.set_fact:
-        nfc_c_mysql_host: "{{ lookup('ansible.builtin.env', 'MYSQL_HOST') | default('') }}"
-        nfc_c_mysql_port: "{{ lookup('ansible.builtin.env', 'MYSQL_PORT') | default(3306) | int }}"
-        nfc_c_mysql_user: "{{ lookup('ansible.builtin.env', 'MYSQL_USER') | default('') }}"
-        nfc_c_mysql_password: "{{ lookup('ansible.builtin.env', 'MYSQL_PASSWORD') | default('') }}"
-      no_log: true
-
-
     - name: TRACE Inbound data Received
       ansible.builtin.debug:
         msg: "{{ inbound_data }}"
 
+    - name: Check for Existance of config file
+      ansible.builtin.stat:
+        path: /etc/phpipam/scan_server.yaml
+      register: config_file_check
+
+
+    - name: Load Config File
+      ansible.builtin.include_vars:
+        file: /etc/phpipam/scan_server.yaml
+      when: >
+        config_file_check.stat.exists
+
 
     - name: Fetch Agent Details
       community.mysql.mysql_query:
-        login_host: "{{ nfc_c_mysql_host }}"
+        login_host: "{{ nofusscomputing_phpipam_scan_server.mysql_host }}"
-        login_port: "{{ nfc_c_mysql_port | default(3306) | int }}"
+        login_port: "{{ nofusscomputing_phpipam_scan_server.mysql_port | default(3306) | int }}"
-        login_user: "{{ nfc_c_mysql_user }}"
+        login_user: "{{ nofusscomputing_phpipam_scan_server.mysql_user }}"
-        login_password: "{{ nfc_c_mysql_password }}"
+        login_password: "{{ nofusscomputing_phpipam_scan_server.mysql_password }}"
 
         login_db: 'phpipam'
         query: >
@@ -35,12 +38,30 @@
       register: mysql_query_agent_details
 
 
+    - name: Update Scan Agent Last seen
+      community.mysql.mysql_query:
+        login_host: "{{ nofusscomputing_phpipam_scan_server.mysql_host }}"
+        login_port: "{{ nofusscomputing_phpipam_scan_server.mysql_port | default(3306) | int }}"
+        login_user: "{{ nofusscomputing_phpipam_scan_server.mysql_user }}"
+        login_password: "{{ nofusscomputing_phpipam_scan_server.mysql_password }}"
+
+        login_db: 'phpipam'
+        query: |-
+          UPDATE
+            scanAgents
+          SET
+          last_access = '{{ ('%Y-%m-%d %H:%M:%S' | strftime) }}'
+          WHERE
+            id = '{{ mysql_query_agent_details.query_result[0][0].id }}'
+        single_transaction: true
+
+
     - name: Confirm Subnet Assignment
       community.mysql.mysql_query:
-        login_host: "{{ nfc_c_mysql_host }}"
+        login_host: "{{ nofusscomputing_phpipam_scan_server.mysql_host }}"
-        login_port: "{{ nfc_c_mysql_port | default(3306) | int }}"
+        login_port: "{{ nofusscomputing_phpipam_scan_server.mysql_port | default(3306) | int }}"
-        login_user: "{{ nfc_c_mysql_user }}"
+        login_user: "{{ nofusscomputing_phpipam_scan_server.mysql_user }}"
-        login_password: "{{ nfc_c_mysql_password }}"
+        login_password: "{{ nofusscomputing_phpipam_scan_server.mysql_password }}"
 
         login_db: 'phpipam'
         query: >

playbooks/tasks/agent_id.yaml

@@ -4,10 +4,10 @@
   ansible.builtin.include_tasks:
     file: tasks/api_call.yaml
   vars:
-    api_client_name: "{{ client_name }}"
+    api_client_name: "{{ nofusscomputing_phpipam_scan_agent.client_name }}"
-    api_token: "{{ client_token }}"
+    api_token: "{{ nofusscomputing_phpipam_scan_agent.client_token }}"
     api_path: "{{ api_scanagents }}"
-    api_query_string: "filter_by=code&filter_value={{ scanagent_code }}"
+    api_query_string: "filter_by=code&filter_value={{ nofusscomputing_phpipam_scan_agent.scanagent_code }}"
 
 
 - name: My ScanAgent ID

playbooks/tasks/api_call.yaml

@@ -3,15 +3,6 @@
   block:
 
 
-    - name: Mandatory Variables set
-      ansible.builtin.assert:
-        that:
-          - api_client_name is defined
-          - api_path is defined
-          - api_token is defined
-          - api_url is defined
-
-
     - name: API Facts
       ansible.builtin.set_fact:
         epoch: "{{ ((('%Y-%m-%d %H:%M:%S' | strftime) | string | to_datetime) - ('1970-01-01 00:00:00' | to_datetime)).total_seconds() | int }}"
@@ -26,64 +17,69 @@
     - name: check Cache Files
       ansible.builtin.stat:
         path: "{{ cache_filepath }}"
-      register: cache_files
+      register: cached_file
 
 
     - name: Expire
       ansible.builtin.set_fact:
-        expired: "{{ ((epoch | int + (nfc_c_epoch_time_offset | default(0)) | int) >= ((cache_files.stat.mtime | int) + nfc_c_cache_expire_time | int) | int ) | bool }}"
+        expired: "{{ ((epoch | int + (nfc_c_epoch_time_offset | default(0)) | int) >= ((cached_file.stat.mtime | int) +
-      when: cache_files.stat.exists
+          (nofusscomputing_phpipam_scan_agent.cache_expire_time | default(nfc_c_cache_expire_time)) | int) | int ) | bool }}"
+      when: cached_file.stat.exists
 
 
     - name: TRACE - Cached file
       ansible.builtin.debug:
         msg:
-          - "exists: {{ cache_files.stat.exists | default('') }}"
+          - "exists: {{ cached_file.stat.exists | default('') }}"
-          - "mtime:  {{ cache_files.stat.mtime | default(0) | int }}"
+          - "mtime:  {{ cached_file.stat.mtime | default(0) | int }}"
-          - "expire: {{ (cache_files.stat.mtime | int) + nfc_c_cache_expire_time | int }}" 
+          - "expire: {{ (cached_file.stat.mtime | int) + (nofusscomputing_phpipam_scan_agent.cache_expire_time | default(nfc_c_cache_expire_time)) | int }}"
-          - "epoch:  {{ (epoch | int + (nfc_c_epoch_time_offset | default(0)) | int) | int }} [{{ nfc_c_cache_expire_time }}]"
+          - "epoch:  {{ (epoch | int + (nfc_c_epoch_time_offset | default(0)) | int) | int }} [{{
+            (nofusscomputing_phpipam_scan_agent.cache_expire_time | default(nfc_c_cache_expire_time)) }}]"
           - "epoch: {{ epoch }}"
           - "expired: {{ expired }}"
-      when: cache_files.stat.exists
+      when: cached_file.stat.exists
 
     - name: Expire Cache
       ansible.builtin.file:
-        path: "{{ cache_files.stat.path }}"
+        path: "{{ cached_file.stat.path }}"
         state: absent
       when: >
         expired
           and
-        cache_files.stat.exists
+        cached_file.stat.exists
 
 
+      # Note: Dont edit http_agent version as the build pipeline updates automagically!!
+      #       see ci variable 'RELEASE_ADDITIONAL_ACTIONS_BUMP'
     - name: >
         PHPIPAM API Call - {{ api_path }}{%- if api_query_string is defined -%}
             /?{{ api_query_string }}
           {%- endif %}
       ansible.builtin.uri:
         url: >-
-          {{ api_url }}/api/{{ api_client_name }}/{{ api_path }}
+          {{ nofusscomputing_phpipam_scan_agent.api_url }}/api/{{ api_client_name }}/{{ api_path }}
           {%- if api_query_string is defined -%}
             /?{{ api_query_string }}
           {%- endif %}
         headers:
           token: "{{ api_token }}"
+        http_agent: nfc-phpipam-scan-agent/0.4.0
         return_content: true
         status_code:
           - 200
           - 404
         validate_certs: false
-      changed_when: api_call.json | length | int > 0
+      changed_when: api_call.json | default([]) | length | int > 0
       no_log: true
       register: api_call
       when: >
         (
           expired
             and
-          cache_files.stat.exists
+          cached_file.stat.exists
         )
           or
-        not cache_files.stat.exists
+        not cached_file.stat.exists
 
 
     - name: Create Cache DIR
@@ -104,9 +100,14 @@
         (
           expired
             and
-          cache_files.stat.exists
+          cached_file.stat.exists
         )
           or
-        not cache_files.stat.exists
+        not cached_file.stat.exists
           and
         api_call.status | default(0) | int != 404
+
+    - name: check Cache Files
+      ansible.builtin.stat:
+        path: "{{ cache_filepath }}"
+      register: cached_file

playbooks/tasks/scan_subnet.yaml

@@ -1,30 +1,82 @@
 ---
 
-- name: Scan subnet
-  ansible.builtin.command:
-    cmd: nmap -sn "{{ subnet.address }}" -oX -
-  become: true
-  register: nmap_scan
 - name: Get subnets Address'
   ansible.builtin.include_tasks:
     file: tasks/api_call.yaml
   vars:
-    api_client_name: "{{ client_name }}"
+    api_client_name: "{{ nofusscomputing_phpipam_scan_agent.client_name }}"
-    api_token: "{{ client_token }}"
+    api_token: "{{ nofusscomputing_phpipam_scan_agent.client_token }}"
     api_path: "{{ api_address }}"
     api_query_string: "filter_by=subnetId&filter_value={{ subnet.id }}"
 
 
-- name: Load Subnet
+- name: Register Subnet API Call
   ansible.builtin.set_fact:
-    cached_subnet: "{{ lookup('file', cache_filepath) }}"
+    cacheable: false
+    subnet_api_call: "{{ api_call }}"
+    subnet_cache_filepath: "{{ cache_filepath }}"
+    api_call: ''
+
+
+- name: Get subnet Name Servers
+  ansible.builtin.include_tasks:
+    file: tasks/api_call.yaml
+  vars:
+    api_client_name: "{{ nofusscomputing_phpipam_scan_agent.client_name }}"
+    api_token: "{{ nofusscomputing_phpipam_scan_agent.client_token }}"
+    api_path: "{{ api_nameservers }}"
+    api_query_string: "filter_by=id&filter_value={{ subnet.nameserverId }}"
+  when: >
+    subnet.nameserverId is defined
+      and
+    subnet.resolveDNS | int == 1
+
+
+- name: Load Nameservers - {{ subnet.address }}
+  ansible.builtin.set_fact:
+    subnet_name_servers: "{{ lookup('file', cache_filepath) }}"
     cacheable: false
   no_log: true
   when: >
+    subnet.nameserverId is defined
+      and
+    subnet.resolveDNS | int == 1
+      and
     api_call.status | default(0) | int != 404
 
 
-- name: Process Scan Results
+- name: Scan subnet - {{ subnet.address }}
+  ansible.builtin.command:
+    cmd: >
+      nmap -sn "{{ subnet.address }}" {% if
+        subnet_name_servers is defined
+          and
+        subnet.resolveDNS | int == 1
+      -%}
+      --dns-servers {% for nameserver in subnet_name_servers -%}
+        {% for name_server in nameserver.namesrv1 | split(';') %}
+          {{ name_server }},
+        {%- endfor -%}
+      {%- endfor -%}
+      {%- elif subnet.resolveDNS | int == 1 -%}
+        --system-dns
+      {%- else -%}
+        -n
+      {%- endif %} -oX -
+  become: true
+  register: nmap_scan
+
+
+- name: Load Subnet - {{ subnet.address }}
+  ansible.builtin.set_fact:
+    cached_subnet: "{{ lookup('file', subnet_cache_filepath) }}"
+    cacheable: false
+  no_log: true
+  when: >
+    subnet_api_call.status | default(0) | int != 404
+
+
+- name: Process Scan Results - {{ subnet.address }}
   ansible.builtin.set_fact:
     subnet_scan_results: |-
       [
@@ -42,29 +94,88 @@
             {%- endfor %}
             "subnetId": "{{ subnet.id }}",
             "ip": "{{ scanned_host.address['@addr'] | default(scanned_host.address[0]['@addr']) }}",
-            "lastSeen": "{{ nmap_scan.start }}",
+            "lastSeen": "{{ (nmap_scan.start | split('.'))[0] }}",
+
+            {% if scanned_host.hostnames.hostname is defined %}
+
+              {% set ip_address = scanned_host.address['@addr'] | default(scanned_host.address[0]['@addr']) | split('.') %}
+
+              {% if
+                '.' in scanned_host.hostnames.hostname['@name'] | string
+                  and
+                (ip_address[0] + '-' + ip_address[1] + '-' + ip_address[2] + '-' + ip_address[3]) not in scanned_host.hostnames.hostname['@name'] | string
+                  and
+                (ip_address[3] + '-' + ip_address[2] + '-' + ip_address[1] + '-' + ip_address[0]) not in scanned_host.hostnames.hostname['@name'] | string
+              %}
+
+                "hostname": "{{ (scanned_host.hostnames.hostname['@name'] | split('.'))[0] }}",
+
+              {% elif
+                '.' not in scanned_host.hostnames.hostname['@name'] | string
+                  and
+                (ip_address[0] + '-' + ip_address[1] + '-' + ip_address[2] + '-' + ip_address[3]) not in scanned_host.hostnames.hostname['@name'] | string
+                  and
+                (ip_address[3] + '-' + ip_address[2] + '-' + ip_address[1] + '-' + ip_address[0]) not in scanned_host.hostnames.hostname['@name'] | string
+              %}
+
+                "hostname": "{{ scanned_host.hostnames.hostname['@name'] }}",
+
+              {% endif %}
+
+            {% endif %}
+
             {% if scanned_host.address['@addrtype'] | default(scanned_host.address[1]['@addrtype']) == 'mac' %}
+
               "mac": "{{ scanned_host.address['@addr'] | default(scanned_host.address[1]['@addr']) | upper }}"
+
             {% endif %}
           },
         {% endif %}
       {% endfor %}
       ]
 
-- name: To JSON
+
+- name: Force Failure for non-HTTPS Communication
+  ansible.builtin.assert:
+    that:
+      - |-
+        not
+          (
+            (
+              'http:' in (nofusscomputing_phpipam_scan_agent.http_server | default(nfc_c_http_server) | string)
+                and
+              'http://127.0.0.1' not in (nofusscomputing_phpipam_scan_agent.http_server | default(nfc_c_http_server) | string)
+            )
+              and
+            nofusscomputing_phpipam_scan_agent.auth_token | default('no-token-set') != 'no-token-set'
+          )
+    fail_msg: 'Failing task as an attempt was made to communicate with the server over a non-encrypted channel'
+    success_msg: 'OK'
+
+
+- name: To JSON - {{ subnet.address }}
   ansible.builtin.set_fact:
     subnet_scan_results: "{{ subnet_scan_results | from_yaml }}"
 
 
-- name: Upload Scan Results
+  # Note: Dont edit http_agent version as the build pipeline updates automagically!!
+  #       see ci variable 'RELEASE_ADDITIONAL_ACTIONS_BUMP'
+- name: Upload Scan Results - {{ subnet.address }}
   ansible.builtin.uri:
-    url: "{{ nfc_c_http_server }}:{{ nfc_c_http_port }}/"
+    headers:
+      Authorization: "Bearer {{ nofusscomputing_phpipam_scan_agent.auth_token | default('no-token-set') }}"
+    http_agent: nfc-phpipam-scan-agent/0.4.0
+    url: "{{
+      nofusscomputing_phpipam_scan_agent.http_server | default(nfc_c_http_server)
+      }}:{{ nofusscomputing_phpipam_scan_agent.http_port | default(nfc_c_http_port) }}/"
     method: POST
     body_format: json
     body: {
-      "code": "{{ scanagent_code }}",
+      "code": "{{ nofusscomputing_phpipam_scan_agent.scanagent_code }}",
       "scan": {
         "subnet": "{{ subnet.address }}",
-        "results": "{{ subnet_scan_results }}"
+        "results": "{{ subnet_scan_results }}",
+        "tz": "{{ '%z' | strftime }}"
       }
     }
+    validate_certs: true    # Ensure always true

playbooks/tasks/server/ipaddress.yaml

@@ -1,12 +1,48 @@
 ---
 
+- name: Convert Scan Time to Local Time
+  ansible.builtin.set_fact:
+    scan_time: |-
+      {%- if inbound_data.scan.tz != ('%z' | strftime | string) -%}
 
-- name: Update IP Address' found
+        {%- set adjust_time = scan_address.ipaddress.lastSeen -%}
+
+        {%- if ('%z' | strftime)[0:1] == '+' -%}
+
+          {{
+            '%Y-%m-%d %H:%M:%S' | strftime(
+              (adjust_time | to_datetime).strftime('%s') | int -
+                (
+                  (inbound_data.scan.tz[1:3] | int * 3600) + (inbound_data.scan.tz[3:5] | int * 60)
+                ) | int
+              )
+          }}
+
+        {%- else -%}
+
+          {{
+            '%Y-%m-%d %H:%M:%S' | strftime((adjust_time | to_datetime).strftime('%s') | int +
+            (
+              (inbound_data.scan.tz[1:3] | int * 3600) + (inbound_data.scan.tz[3:5] | int * 60)
+            ) | int)
+          }}
+
+        {%- endif -%}
+
+      {%- else -%}
+
+        {{ scan_address.ipaddress.lastSeen }}
+
+      {%- endif %}
+    cacheable: false
+
+
+- name: "Update IP Address' found - {{ scan_address.ipaddress.ip }}"
   community.mysql.mysql_query:
-    login_host: "{{ nfc_c_mysql_host }}"
+    login_host: "{{ nofusscomputing_phpipam_scan_server.mysql_host }}"
-    login_port: "{{ nfc_c_mysql_port | default(3306) | int }}"
+    login_port: "{{ nofusscomputing_phpipam_scan_server.mysql_port | default(3306) | int }}"
-    login_user: "{{ nfc_c_mysql_user }}"
+    login_user: "{{ nofusscomputing_phpipam_scan_server.mysql_user }}"
-    login_password: "{{ nfc_c_mysql_password }}"
+    login_password: "{{ nofusscomputing_phpipam_scan_server.mysql_password }}"
 
     login_db: 'phpipam'
     query: |-
@@ -16,7 +52,13 @@
 
         UPDATE ipaddresses
         SET
-          lastSeen = '{{ scan_address.ipaddress.lastSeen }}'
+          lastSeen = '{{ scan_time }}'
+
+            {% if scan_address.ipaddress.hostname | default('') != '' %},
+
+            hostname = '{{ scan_address.ipaddress.hostname }}'
+
+            {% endif %}
 
           {% if scan_address.ipaddress.mac | default('') != '' %},
 
@@ -36,6 +78,7 @@
             subnetId,
             ip_addr,
             description,
+            {% if scan_address.ipaddress.hostname | default('') != '' %}hostname,{% endif %}
             {% if scan_address.ipaddress.mac | default('') != '' %}mac,{% endif %}
             note,
             lastSeen
@@ -46,6 +89,12 @@
             '{{ scan_address.ipaddress.ip | ip2ipam }}',
             '-- autodiscovered --',
 
+            {% if scan_address.ipaddress.hostname | default('') != '' %}
+
+              '{{ scan_address.ipaddress.hostname }}',
+
+            {% endif %}
+
             {% if scan_address.ipaddress.mac | default('') != '' %}
 
               '{{ scan_address.ipaddress.mac }}',

playbooks/tasks/server/subnet_scan.yaml

@@ -3,10 +3,10 @@
 
 - name: Match Scan Addresses to DB Details
   community.mysql.mysql_query:
-    login_host: "{{ nfc_c_mysql_host }}"
+    login_host: "{{ nofusscomputing_phpipam_scan_server.mysql_host }}"
-    login_port: "{{ nfc_c_mysql_port | default(3306) | int }}"
+    login_port: "{{ nofusscomputing_phpipam_scan_server.mysql_port | default(3306) | int }}"
-    login_user: "{{ nfc_c_mysql_user }}"
+    login_user: "{{ nofusscomputing_phpipam_scan_server.mysql_user }}"
-    login_password: "{{ nfc_c_mysql_password }}"
+    login_password: "{{ nofusscomputing_phpipam_scan_server.mysql_password }}"
 
     login_db: 'phpipam'
     query: |-
@@ -26,6 +26,24 @@
     loop_var: ipaddress
 
 
+- name: Update Subnet Discovery Time
+  community.mysql.mysql_query:
+    login_host: "{{ nofusscomputing_phpipam_scan_server.mysql_host }}"
+    login_port: "{{ nofusscomputing_phpipam_scan_server.mysql_port | default(3306) | int }}"
+    login_user: "{{ nofusscomputing_phpipam_scan_server.mysql_user }}"
+    login_password: "{{ nofusscomputing_phpipam_scan_server.mysql_password }}"
+
+    login_db: 'phpipam'
+    query: |-
+      UPDATE
+        subnets
+      SET
+       lastDiscovery = '{{ ('%Y-%m-%d %H:%M:%S' | strftime) }}'
+      WHERE
+        subnet = '{{ (inbound_data.scan.subnet | split('/'))[0] | ip2ipam }}'
+    single_transaction: true
+
+
 - name: Update IP Addresses
   ansible.builtin.include_tasks:
     file: tasks/server/ipaddress.yaml

playbooks/tasks/subnets.yaml

@@ -9,21 +9,35 @@
   ansible.builtin.include_tasks:
     file: tasks/api_call.yaml
   vars:
-    api_client_name: "{{ client_name }}"
+    api_client_name: "{{ nofusscomputing_phpipam_scan_agent.client_name }}"
-    api_token: "{{ client_token }}"
+    api_token: "{{ nofusscomputing_phpipam_scan_agent.client_token }}"
     api_path: "{{ api_subnets }}"
     api_query_string: "filter_by=scanAgent&filter_value={{ nfc_c_scan_agent_id }}"
 
 
-- name: Update Subnets List
+- name: Try/Catch
-  ansible.builtin.set_fact:
+  block:
-    nfc_c_scan_agent_subnets: "{{ nfc_c_scan_agent_subnets + [{
+
-        'id': network.id,
+
-        'address': network.subnet + '/' + network.mask
+    - name: Update Subnets List
-      }] }}"
+      ansible.builtin.set_fact:
-  loop: "{{ data | default ([]) }}"
+        nfc_c_scan_agent_subnets: "{{ nfc_c_scan_agent_subnets + [{
-  loop_control:
+            'id': network.id,
-    loop_var: network
+            'address': network.subnet + '/' + network.mask,
-  vars:
+            'nameserverId': network.nameserverId,
-    data: "{{ lookup('file', cache_filepath) }}"
+            'resolveDNS': network.resolveDNS 
-  when: network.discoverSubnet | int == 1
+          }] }}"
+      loop: "{{ lookup('file', cache_filepath) | default ([]) }}"
+      loop_control:
+        loop_var: network
+      when: >
+        network.discoverSubnet | int == 1
+
+  rescue:
+
+    - name: Confirm 'Subnets List' Error is Expected
+      ansible.builtin.assert:
+        that:
+          - not cached_file.stat.exists
+        success_msg: "OK. Success. The error occured as there is no cache file. This normally means there are no subnets assigned to the agent."
+        fail_msg: "ERROR. Something went wrong, Cache file exists."