diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 89bdc43..8d3e5a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,18 +19,8 @@ variables: DOCKER_IMAGE_PUBLISH_REGISTRY: docker.io/nofusscomputing DOCKER_IMAGE_PUBLISH_URL: https://hub.docker.com/r/nofusscomputing/$DOCKER_IMAGE_PUBLISH_NAME - RELEASE_ADDITIONAL_ACTIONS_BUMP: | - sed -E "/http_agent: nfc-phpipam-scan-agent/s/\/(.+)/\/$(cz version --project)/g" -i playbooks/tasks/scan_subnet.yaml; + RELEASE_ADDITIONAL_ACTIONS_BUMP: ./.gitlab/additional_actions_bump.sh - git add playbooks/tasks/scan_subnet.yaml; - - git status; - - sed -E "/http_agent: nfc-phpipam-scan-agent/s/\/(.+)/\/$(cz version --project)/g" -i playbooks/tasks/api_call.yaml - - git add playbooks/tasks/api_call.yaml; - - git status; include: @@ -40,6 +30,7 @@ include: ref: development file: - .gitlab-ci_common.yaml + - conventional_commits/.gitlab-ci.yml - template/ansible-collection.gitlab-ci.yaml - template/mkdocs-documentation.gitlab-ci.yaml # ToDo: update gitlabCI jobs for collections workflow diff --git a/.gitlab/additional_actions_bump.sh b/.gitlab/additional_actions_bump.sh new file mode 100644 index 0000000..4c8bbb2 --- /dev/null +++ b/.gitlab/additional_actions_bump.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +sed -E "/http_agent: nfc-phpipam-scan-agent/s/\/(.+)/\/$(cz version --project)/g" -i playbooks/tasks/scan_subnet.yaml; + +git add playbooks/tasks/scan_subnet.yaml; + +git status; + +sed -E "/http_agent: nfc-phpipam-scan-agent/s/\/(.+)/\/$(cz version --project)/g" -i playbooks/tasks/api_call.yaml; + +git add playbooks/tasks/api_call.yaml; + +git status; diff --git a/docs/projects/ansible/collection/phpipam_scan_agent/index.md b/docs/projects/ansible/collection/phpipam_scan_agent/index.md index b94082d..82bc806 100644 --- a/docs/projects/ansible/collection/phpipam_scan_agent/index.md +++ b/docs/projects/ansible/collection/phpipam_scan_agent/index.md @@ -32,6 +32,8 @@ This collection is available on Ansible Galaxy and can be installed with `ansibl Prefer to use our [docker](docker.md) image? It's available on Docker Hub `docker pull nofusscomputing/phpipam-scan-agent:latest`. +There is no timezone support within phpIPAM. Due to this it's recommended that you setup phpIPAM, the database and the scanner component to use UTC (GMT +00:00) timezone or that all components be setup to use the same timezone. + ## Features @@ -50,9 +52,11 @@ The following features are available or planned to be implmented: !!! info It's only possible to obtain a MAC Address if the scanner is on the same L2 network (Broadcast Domain). Within the docs you will find the different methods available to achieve this. -- Remote Network Scanning +- [Remote Network Scanning](scanner.md#remote-network-scannning) -- [**ToDo** Resolve DNS names](https://gitlab.com/nofusscomputing/projects/ansible/collections/phpipam_scan_agent/-/issues/4) +- [Resolve DNS names](index.md#Resolve-dns-names) + +- [Timezone Normalization](server.md#timezone-normalization) ## phpIPAM Features @@ -75,6 +79,16 @@ When this setting is enabled, the scanner will scan the entire subnet that has b This is displayed within the subnet interface next to the `Discover new hosts` field. Whenever a scan report is received by the server this filed is updated to reflect the last scan time. +### Resolve DNS names + +- Location `Subnet -> Resolve DNS names` + +When this setting is enabled, DNS resolution for the hostname for any IP found will be conducted. If you have setup nameservers (`Administration -> Nameservers`) and assigned them to the subnet, they will be used for name resolution. If no nameservers are configured for the subnet, the host's nameservers where the scanner is running from will be used. BY default within phpIPAM, DNS Resolution is turned off for a subnet and as such the scanner will not resolve IP address DNS names. + +!!! info + The FQDN for an IP that is retured is not used as the hostname. **Only** the first octet. i.e. if the reverse DNS lookup returns a FQDN of `my_hostname.domainname.tld`, the hostname for the IP address will be set to `my_hostname` + + ### Scan Agent - Location `Administration -> Scan agents [Last access]` diff --git a/docs/projects/ansible/collection/phpipam_scan_agent/server.md b/docs/projects/ansible/collection/phpipam_scan_agent/server.md index 4992b9f..1c68b86 100644 --- a/docs/projects/ansible/collection/phpipam_scan_agent/server.md +++ b/docs/projects/ansible/collection/phpipam_scan_agent/server.md @@ -67,3 +67,7 @@ The [scan](scanner.md#remote-network-scannning) and server component must be set !!! danger "Security" Failing to secure the server component communication with TLS will allow anyone with direct access to the line of communication to view the `auth_token`. Anyone who has the `auth_token` will be able to upload data to the server. + +## Timezone normalization + +As there is no support within phpIPAM for different timezones. The server component when receiving updates from scanners, will convert any time found to UTC (GMT +00:00). This is required so that phpIPAM features that rely on time, function as they should. This setup requires that machine or docker containers for phpIPAM and the MySQL/MariaDB database both have their timezones set to UTC. If you don't wish for any timezone conversion to be done, ensure that where ever all components, including the scan server component, share the same timezone. diff --git a/gitlab-ci b/gitlab-ci index 6f80ea3..41eeb7b 160000 --- a/gitlab-ci +++ b/gitlab-ci @@ -1 +1 @@ -Subproject commit 6f80ea3af7fdc64e9998820a8800c288d7facbc6 +Subproject commit 41eeb7badd582175b371cd4a5b2192decbcb0210 diff --git a/playbooks/agent.yaml b/playbooks/agent.yaml index a3f4293..32c29c9 100644 --- a/playbooks/agent.yaml +++ b/playbooks/agent.yaml @@ -82,3 +82,4 @@ api_address: addresses api_subnets: subnets api_scanagents: tools/scanagents + api_nameservers: tools/nameservers diff --git a/playbooks/tasks/scan_subnet.yaml b/playbooks/tasks/scan_subnet.yaml index b63fefb..1e92c6a 100644 --- a/playbooks/tasks/scan_subnet.yaml +++ b/playbooks/tasks/scan_subnet.yaml @@ -1,12 +1,5 @@ --- -- name: Scan subnet - {{ subnet.address }} - ansible.builtin.command: - cmd: nmap -sn "{{ subnet.address }}" -oX - - become: true - register: nmap_scan - - - name: Get subnets Address' ansible.builtin.include_tasks: file: tasks/api_call.yaml @@ -17,15 +10,72 @@ api_query_string: "filter_by=subnetId&filter_value={{ subnet.id }}" -- name: Load Subnet - {{ subnet.address }} +- name: Register Subnet API Call ansible.builtin.set_fact: - cached_subnet: "{{ lookup('file', cache_filepath) }}" + cacheable: false + subnet_api_call: "{{ api_call }}" + subnet_cache_filepath: "{{ cache_filepath }}" + api_call: '' + + +- name: Get subnet Name Servers + ansible.builtin.include_tasks: + file: tasks/api_call.yaml + vars: + api_client_name: "{{ nofusscomputing_phpipam_scan_agent.client_name }}" + api_token: "{{ nofusscomputing_phpipam_scan_agent.client_token }}" + api_path: "{{ api_nameservers }}" + api_query_string: "filter_by=id&filter_value={{ subnet.nameserverId }}" + when: > + subnet.nameserverId is defined + and + subnet.resolveDNS | int == 1 + + +- name: Load Nameservers - {{ subnet.address }} + ansible.builtin.set_fact: + subnet_name_servers: "{{ lookup('file', cache_filepath) }}" cacheable: false no_log: true when: > + subnet.nameserverId is defined + and + subnet.resolveDNS | int == 1 + and api_call.status | default(0) | int != 404 +- name: Scan subnet - {{ subnet.address }} + ansible.builtin.command: + cmd: > + nmap -sn "{{ subnet.address }}" {% if + subnet_name_servers is defined + and + subnet.resolveDNS | int == 1 + -%} + --dns-servers {% for nameserver in subnet_name_servers -%} + {% for name_server in nameserver.namesrv1 | split(';') %} + {{ name_server }}, + {%- endfor -%} + {%- endfor -%} + {%- elif subnet.resolveDNS | int == 1 -%} + --system-dns + {%- else -%} + -n + {%- endif %} -oX - + become: true + register: nmap_scan + + +- name: Load Subnet - {{ subnet.address }} + ansible.builtin.set_fact: + cached_subnet: "{{ lookup('file', subnet_cache_filepath) }}" + cacheable: false + no_log: true + when: > + subnet_api_call.status | default(0) | int != 404 + + - name: Process Scan Results - {{ subnet.address }} ansible.builtin.set_fact: subnet_scan_results: |- diff --git a/playbooks/tasks/server/ipaddress.yaml b/playbooks/tasks/server/ipaddress.yaml index fb80058..1d44304 100644 --- a/playbooks/tasks/server/ipaddress.yaml +++ b/playbooks/tasks/server/ipaddress.yaml @@ -1,5 +1,41 @@ --- +- name: Convert Scan Time to Local Time + ansible.builtin.set_fact: + scan_time: |- + {%- if inbound_data.scan.tz != ('%z' | strftime | string) -%} + + {%- set adjust_time = scan_address.ipaddress.lastSeen -%} + + {%- if ('%z' | strftime)[0:1] == '+' -%} + + {{ + '%Y-%m-%d %H:%M:%S' | strftime( + (adjust_time | to_datetime).strftime('%s') | int - + ( + (inbound_data.scan.tz[1:3] | int * 3600) + (inbound_data.scan.tz[3:5] | int * 60) + ) | int + ) + }} + + {%- else -%} + + {{ + '%Y-%m-%d %H:%M:%S' | strftime((adjust_time | to_datetime).strftime('%s') | int + + ( + (inbound_data.scan.tz[1:3] | int * 3600) + (inbound_data.scan.tz[3:5] | int * 60) + ) | int) + }} + + {%- endif -%} + + {%- else -%} + + {{ scan_address.ipaddress.lastSeen }} + + {%- endif %} + cacheable: false + - name: "Update IP Address' found - {{ scan_address.ipaddress.ip }}" community.mysql.mysql_query: @@ -16,7 +52,7 @@ UPDATE ipaddresses SET - lastSeen = '{{ scan_address.ipaddress.lastSeen }}' + lastSeen = '{{ scan_time }}' {% if scan_address.ipaddress.hostname | default('') != '' %}, diff --git a/playbooks/tasks/server/subnet_scan.yaml b/playbooks/tasks/server/subnet_scan.yaml index c7d4f33..a091ba6 100644 --- a/playbooks/tasks/server/subnet_scan.yaml +++ b/playbooks/tasks/server/subnet_scan.yaml @@ -38,7 +38,7 @@ UPDATE subnets SET - lastDiscovery = '{{ inbound_data.scan.results[0].lastSeen }}' + lastDiscovery = '{{ ('%Y-%m-%d %H:%M:%S' | strftime) }}' WHERE subnet = '{{ (inbound_data.scan.subnet | split('/'))[0] | ip2ipam }}' single_transaction: true diff --git a/playbooks/tasks/subnets.yaml b/playbooks/tasks/subnets.yaml index 785b31c..7ab9324 100644 --- a/playbooks/tasks/subnets.yaml +++ b/playbooks/tasks/subnets.yaml @@ -23,7 +23,9 @@ ansible.builtin.set_fact: nfc_c_scan_agent_subnets: "{{ nfc_c_scan_agent_subnets + [{ 'id': network.id, - 'address': network.subnet + '/' + network.mask + 'address': network.subnet + '/' + network.mask, + 'nameserverId': network.nameserverId, + 'resolveDNS': network.resolveDNS }] }}" loop: "{{ lookup('file', cache_filepath) | default ([]) }}" loop_control: