From 3ed00b733eb9fb13b5d8046d7489d5de6c869ba0 Mon Sep 17 00:00:00 2001 From: Jon Date: Sun, 25 Feb 2024 16:19:59 +0930 Subject: [PATCH 1/6] feat(server): Convert scanned time within scan report to UTC !13 closes #13 --- .../collection/phpipam_scan_agent/index.md | 4 ++ .../collection/phpipam_scan_agent/server.md | 4 ++ playbooks/tasks/server/ipaddress.yaml | 38 ++++++++++++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/docs/projects/ansible/collection/phpipam_scan_agent/index.md b/docs/projects/ansible/collection/phpipam_scan_agent/index.md index b94082d..2dce13f 100644 --- a/docs/projects/ansible/collection/phpipam_scan_agent/index.md +++ b/docs/projects/ansible/collection/phpipam_scan_agent/index.md @@ -32,6 +32,8 @@ This collection is available on Ansible Galaxy and can be installed with `ansibl Prefer to use our [docker](docker.md) image? It's available on Docker Hub `docker pull nofusscomputing/phpipam-scan-agent:latest`. +There is no timezone support within phpIPAM. Due to this it's recommended that you setup phpIPAM, the database and the scanner component to use UTC (GMT +00:00) timezone or that all components be setup to use the same timezone. + ## Features @@ -54,6 +56,8 @@ The following features are available or planned to be implmented: - [**ToDo** Resolve DNS names](https://gitlab.com/nofusscomputing/projects/ansible/collections/phpipam_scan_agent/-/issues/4) +- [Timezone Normalization](server.md#timezone-normalization) + ## phpIPAM Features diff --git a/docs/projects/ansible/collection/phpipam_scan_agent/server.md b/docs/projects/ansible/collection/phpipam_scan_agent/server.md index 4992b9f..1c68b86 100644 --- a/docs/projects/ansible/collection/phpipam_scan_agent/server.md +++ b/docs/projects/ansible/collection/phpipam_scan_agent/server.md @@ -67,3 +67,7 @@ The [scan](scanner.md#remote-network-scannning) and server component must be set !!! danger "Security" Failing to secure the server component communication with TLS will allow anyone with direct access to the line of communication to view the `auth_token`. Anyone who has the `auth_token` will be able to upload data to the server. + +## Timezone normalization + +As there is no support within phpIPAM for different timezones. The server component when receiving updates from scanners, will convert any time found to UTC (GMT +00:00). This is required so that phpIPAM features that rely on time, function as they should. This setup requires that machine or docker containers for phpIPAM and the MySQL/MariaDB database both have their timezones set to UTC. If you don't wish for any timezone conversion to be done, ensure that where ever all components, including the scan server component, share the same timezone. diff --git a/playbooks/tasks/server/ipaddress.yaml b/playbooks/tasks/server/ipaddress.yaml index fb80058..1d44304 100644 --- a/playbooks/tasks/server/ipaddress.yaml +++ b/playbooks/tasks/server/ipaddress.yaml @@ -1,5 +1,41 @@ --- +- name: Convert Scan Time to Local Time + ansible.builtin.set_fact: + scan_time: |- + {%- if inbound_data.scan.tz != ('%z' | strftime | string) -%} + + {%- set adjust_time = scan_address.ipaddress.lastSeen -%} + + {%- if ('%z' | strftime)[0:1] == '+' -%} + + {{ + '%Y-%m-%d %H:%M:%S' | strftime( + (adjust_time | to_datetime).strftime('%s') | int - + ( + (inbound_data.scan.tz[1:3] | int * 3600) + (inbound_data.scan.tz[3:5] | int * 60) + ) | int + ) + }} + + {%- else -%} + + {{ + '%Y-%m-%d %H:%M:%S' | strftime((adjust_time | to_datetime).strftime('%s') | int + + ( + (inbound_data.scan.tz[1:3] | int * 3600) + (inbound_data.scan.tz[3:5] | int * 60) + ) | int) + }} + + {%- endif -%} + + {%- else -%} + + {{ scan_address.ipaddress.lastSeen }} + + {%- endif %} + cacheable: false + - name: "Update IP Address' found - {{ scan_address.ipaddress.ip }}" community.mysql.mysql_query: @@ -16,7 +52,7 @@ UPDATE ipaddresses SET - lastSeen = '{{ scan_address.ipaddress.lastSeen }}' + lastSeen = '{{ scan_time }}' {% if scan_address.ipaddress.hostname | default('') != '' %}, -- 2.49.0 From 93d40b743270b8999cdef9d703520f24766e3c12 Mon Sep 17 00:00:00 2001 From: Jon Date: Sun, 25 Feb 2024 16:22:03 +0930 Subject: [PATCH 2/6] fix(server): Time of last access for scan agent to use 'now' !13 #5 --- playbooks/tasks/server/subnet_scan.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/playbooks/tasks/server/subnet_scan.yaml b/playbooks/tasks/server/subnet_scan.yaml index c7d4f33..a091ba6 100644 --- a/playbooks/tasks/server/subnet_scan.yaml +++ b/playbooks/tasks/server/subnet_scan.yaml @@ -38,7 +38,7 @@ UPDATE subnets SET - lastDiscovery = '{{ inbound_data.scan.results[0].lastSeen }}' + lastDiscovery = '{{ ('%Y-%m-%d %H:%M:%S' | strftime) }}' WHERE subnet = '{{ (inbound_data.scan.subnet | split('/'))[0] | ip2ipam }}' single_transaction: true -- 2.49.0 From f89212b1cbdd68c2371d09c24d94d02aa9e2c2e6 Mon Sep 17 00:00:00 2001 From: Jon Date: Sun, 25 Feb 2024 18:02:13 +0930 Subject: [PATCH 3/6] feat(server): DNS Reverse Lookup for IP Address' !13 closes #4 --- .../collection/phpipam_scan_agent/index.md | 12 +++- playbooks/agent.yaml | 1 + playbooks/tasks/scan_subnet.yaml | 68 ++++++++++++++++--- playbooks/tasks/subnets.yaml | 4 +- 4 files changed, 74 insertions(+), 11 deletions(-) diff --git a/docs/projects/ansible/collection/phpipam_scan_agent/index.md b/docs/projects/ansible/collection/phpipam_scan_agent/index.md index 2dce13f..3fc40f0 100644 --- a/docs/projects/ansible/collection/phpipam_scan_agent/index.md +++ b/docs/projects/ansible/collection/phpipam_scan_agent/index.md @@ -54,7 +54,7 @@ The following features are available or planned to be implmented: - Remote Network Scanning -- [**ToDo** Resolve DNS names](https://gitlab.com/nofusscomputing/projects/ansible/collections/phpipam_scan_agent/-/issues/4) +- [Resolve DNS names](index.md#Resolve-dns-names) - [Timezone Normalization](server.md#timezone-normalization) @@ -79,6 +79,16 @@ When this setting is enabled, the scanner will scan the entire subnet that has b This is displayed within the subnet interface next to the `Discover new hosts` field. Whenever a scan report is received by the server this filed is updated to reflect the last scan time. +### Resolve DNS names + +- Location `Subnet -> Resolve DNS names` + +When this setting is enabled, DNS resolution for the hostname for any IP found will be conducted. If you have setup nameservers (`Administration -> Nameservers`) and assigned them to the subnet, they will be used for name resolution. If no nameservers are configured for the subnet, the host's nameservers where the scanner is running from will be used. BY default within phpIPAM, DNS Resolution is turned off for a subnet and as such the scanner will not resolve IP address DNS names. + +!!! info + The FQDN for an IP that is retured is not used as the hostname. **Only** the first octet. i.e. if the reverse DNS lookup returns a FQDN of `my_hostname.domainname.tld`, the hostname for the IP address will be set to `my_hostname` + + ### Scan Agent - Location `Administration -> Scan agents [Last access]` diff --git a/playbooks/agent.yaml b/playbooks/agent.yaml index a3f4293..32c29c9 100644 --- a/playbooks/agent.yaml +++ b/playbooks/agent.yaml @@ -82,3 +82,4 @@ api_address: addresses api_subnets: subnets api_scanagents: tools/scanagents + api_nameservers: tools/nameservers diff --git a/playbooks/tasks/scan_subnet.yaml b/playbooks/tasks/scan_subnet.yaml index b63fefb..1e92c6a 100644 --- a/playbooks/tasks/scan_subnet.yaml +++ b/playbooks/tasks/scan_subnet.yaml @@ -1,12 +1,5 @@ --- -- name: Scan subnet - {{ subnet.address }} - ansible.builtin.command: - cmd: nmap -sn "{{ subnet.address }}" -oX - - become: true - register: nmap_scan - - - name: Get subnets Address' ansible.builtin.include_tasks: file: tasks/api_call.yaml @@ -17,15 +10,72 @@ api_query_string: "filter_by=subnetId&filter_value={{ subnet.id }}" -- name: Load Subnet - {{ subnet.address }} +- name: Register Subnet API Call ansible.builtin.set_fact: - cached_subnet: "{{ lookup('file', cache_filepath) }}" + cacheable: false + subnet_api_call: "{{ api_call }}" + subnet_cache_filepath: "{{ cache_filepath }}" + api_call: '' + + +- name: Get subnet Name Servers + ansible.builtin.include_tasks: + file: tasks/api_call.yaml + vars: + api_client_name: "{{ nofusscomputing_phpipam_scan_agent.client_name }}" + api_token: "{{ nofusscomputing_phpipam_scan_agent.client_token }}" + api_path: "{{ api_nameservers }}" + api_query_string: "filter_by=id&filter_value={{ subnet.nameserverId }}" + when: > + subnet.nameserverId is defined + and + subnet.resolveDNS | int == 1 + + +- name: Load Nameservers - {{ subnet.address }} + ansible.builtin.set_fact: + subnet_name_servers: "{{ lookup('file', cache_filepath) }}" cacheable: false no_log: true when: > + subnet.nameserverId is defined + and + subnet.resolveDNS | int == 1 + and api_call.status | default(0) | int != 404 +- name: Scan subnet - {{ subnet.address }} + ansible.builtin.command: + cmd: > + nmap -sn "{{ subnet.address }}" {% if + subnet_name_servers is defined + and + subnet.resolveDNS | int == 1 + -%} + --dns-servers {% for nameserver in subnet_name_servers -%} + {% for name_server in nameserver.namesrv1 | split(';') %} + {{ name_server }}, + {%- endfor -%} + {%- endfor -%} + {%- elif subnet.resolveDNS | int == 1 -%} + --system-dns + {%- else -%} + -n + {%- endif %} -oX - + become: true + register: nmap_scan + + +- name: Load Subnet - {{ subnet.address }} + ansible.builtin.set_fact: + cached_subnet: "{{ lookup('file', subnet_cache_filepath) }}" + cacheable: false + no_log: true + when: > + subnet_api_call.status | default(0) | int != 404 + + - name: Process Scan Results - {{ subnet.address }} ansible.builtin.set_fact: subnet_scan_results: |- diff --git a/playbooks/tasks/subnets.yaml b/playbooks/tasks/subnets.yaml index 785b31c..7ab9324 100644 --- a/playbooks/tasks/subnets.yaml +++ b/playbooks/tasks/subnets.yaml @@ -23,7 +23,9 @@ ansible.builtin.set_fact: nfc_c_scan_agent_subnets: "{{ nfc_c_scan_agent_subnets + [{ 'id': network.id, - 'address': network.subnet + '/' + network.mask + 'address': network.subnet + '/' + network.mask, + 'nameserverId': network.nameserverId, + 'resolveDNS': network.resolveDNS }] }}" loop: "{{ lookup('file', cache_filepath) | default ([]) }}" loop_control: -- 2.49.0 From 82b8f535bbbe6435b459e38ac9b0b69781a6f868 Mon Sep 17 00:00:00 2001 From: Jon Date: Sun, 25 Feb 2024 18:14:16 +0930 Subject: [PATCH 4/6] docs(index): add link to scanner docs for remote scanning !13 --- docs/projects/ansible/collection/phpipam_scan_agent/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/projects/ansible/collection/phpipam_scan_agent/index.md b/docs/projects/ansible/collection/phpipam_scan_agent/index.md index 3fc40f0..82bc806 100644 --- a/docs/projects/ansible/collection/phpipam_scan_agent/index.md +++ b/docs/projects/ansible/collection/phpipam_scan_agent/index.md @@ -52,7 +52,7 @@ The following features are available or planned to be implmented: !!! info It's only possible to obtain a MAC Address if the scanner is on the same L2 network (Broadcast Domain). Within the docs you will find the different methods available to achieve this. -- Remote Network Scanning +- [Remote Network Scanning](scanner.md#remote-network-scannning) - [Resolve DNS names](index.md#Resolve-dns-names) -- 2.49.0 From b0619f4b8f3410cd734e84e97b45e422ac81dede Mon Sep 17 00:00:00 2001 From: Jon Date: Sun, 25 Feb 2024 19:01:15 +0930 Subject: [PATCH 5/6] fix(ci): http user_agent version set during version bump !13 fixes #14 --- .gitlab-ci.yml | 12 +----------- .gitlab/additional_actions_bump.sh | 13 +++++++++++++ gitlab-ci | 2 +- 3 files changed, 15 insertions(+), 12 deletions(-) create mode 100644 .gitlab/additional_actions_bump.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 89bdc43..5cdac04 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,18 +19,8 @@ variables: DOCKER_IMAGE_PUBLISH_REGISTRY: docker.io/nofusscomputing DOCKER_IMAGE_PUBLISH_URL: https://hub.docker.com/r/nofusscomputing/$DOCKER_IMAGE_PUBLISH_NAME - RELEASE_ADDITIONAL_ACTIONS_BUMP: | - sed -E "/http_agent: nfc-phpipam-scan-agent/s/\/(.+)/\/$(cz version --project)/g" -i playbooks/tasks/scan_subnet.yaml; + RELEASE_ADDITIONAL_ACTIONS_BUMP: ./.gitlab/additional_actions_bump.sh - git add playbooks/tasks/scan_subnet.yaml; - - git status; - - sed -E "/http_agent: nfc-phpipam-scan-agent/s/\/(.+)/\/$(cz version --project)/g" -i playbooks/tasks/api_call.yaml - - git add playbooks/tasks/api_call.yaml; - - git status; include: diff --git a/.gitlab/additional_actions_bump.sh b/.gitlab/additional_actions_bump.sh new file mode 100644 index 0000000..4c8bbb2 --- /dev/null +++ b/.gitlab/additional_actions_bump.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +sed -E "/http_agent: nfc-phpipam-scan-agent/s/\/(.+)/\/$(cz version --project)/g" -i playbooks/tasks/scan_subnet.yaml; + +git add playbooks/tasks/scan_subnet.yaml; + +git status; + +sed -E "/http_agent: nfc-phpipam-scan-agent/s/\/(.+)/\/$(cz version --project)/g" -i playbooks/tasks/api_call.yaml; + +git add playbooks/tasks/api_call.yaml; + +git status; diff --git a/gitlab-ci b/gitlab-ci index 6f80ea3..41eeb7b 160000 --- a/gitlab-ci +++ b/gitlab-ci @@ -1 +1 @@ -Subproject commit 6f80ea3af7fdc64e9998820a8800c288d7facbc6 +Subproject commit 41eeb7badd582175b371cd4a5b2192decbcb0210 -- 2.49.0 From 1e8f3901d42eb238894e07ccb2ddebbbb7bdd81b Mon Sep 17 00:00:00 2001 From: Jon Date: Sun, 25 Feb 2024 19:03:36 +0930 Subject: [PATCH 6/6] ci: add commit validation !13 --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5cdac04..8d3e5a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,6 +30,7 @@ include: ref: development file: - .gitlab-ci_common.yaml + - conventional_commits/.gitlab-ci.yml - template/ansible-collection.gitlab-ci.yaml - template/mkdocs-documentation.gitlab-ci.yaml # ToDo: update gitlabCI jobs for collections workflow -- 2.49.0