Fetching data from endpoint, retrieving coordinates, then merging into an output

Question

I'm in no way a skilled developer (or even a developer at that!) but I did want to take a go at writing a script that fetched one endpoint's data, extracted some data, sent if off to be geocoded, then saved both the original (with geocodes) and the geocoded data to file. On subsequent requests, it would look up the geocoded data on file to check if anything matched, and pull data from there, otherwise trigger the fetch request again.

I think it works - at least it's been working from what I can tell but I'd love a review (or improvement if that's your thing) :)

#!/usr/bin/env node
'use strict';
//
// MARK: - modules
//
const fs = require('fs');
const fetch = require('node-fetch');
//
// MARK: create the variables / constants
//
// get the command line arguments
const args = (function(argv) {
 // remove `node` and `script` name
 argv = argv.slice(2);
 // returned object
 var args = {};
 var argName, argValue;
 // loop through each argument
 argv.forEach(function(arg, index) {
 // seperate argument, for a key/value return
 arg = arg.split('=');
 // retrieve the argument name
 argName = arg[0];
 // remove "--" or "-"
 if (argName.indexOf('-') === 0) {
 argName = argName.slice(
 argName.slice(0, 2).lastIndexOf('-') + 1
 );
 }
 // associate defined value or initialize it to "true" state
 argValue = (arg.length === 2) ?
 // check if argument is valid number
 parseFloat(arg[1]).toString() === arg[1]
 ?
 +arg[1]: arg[1]
 :
 true;
 // finally add the argument to the args set
 args[argName] = argValue;
 });
 return args;
})(process.argv);
// api key checks
const apiKey = function() {
 // api not declared
 if (!args.api) {
 return 1;
 }
 // declared but empty
 if (args.api === true) {
 return 2;
 }
 // all good
 if (args.api && args.api !== true) {
 return args.api;
 }
}(args);
//
// MARK: - check the files and directories we need
//
const req = (() => {
 // where we store things
 const directory = './docs';
 const datavic = `${ directory }/datavic.json`;
 const database = `./database.json`;
 // checks and creation
 if (!fs.existsSync(directory)) {
 fs.mkdirSync(directory);
 }
 // array of items
 return {
 dir: directory,
 api: datavic,
 db: database
 }
})();
//
// MARK: - the script initialiser
//
function init() {
 console.time();
 fetchExposures();
 console.timeEnd();
}
//
// MARK: - 1. fetch the data from datavic
//
function fetchExposures() {
 // api check
 if (apiKey === 1) {
 return console.error(
 '[x] Exiting: API argument not declared!'
 );
 }
 if (apiKey === 2) {
 return console.error(
 '[x] Exiting: API key not provided!'
 );
 }
 // url: Data Victoria API
 const url = new URL(
 'https://discover.data.vic.gov.au/api/3/action/datastore_search'
 );
 // url: parameters
 url.searchParams.append(
 'resource_id', 'afb52611-6061-4a2b-9110-74c920bede77'
 );
 url.searchParams.append(
 'limit', '10000'
 );
 // fetch the data
 fetch(url)
 .then(res => res.json())
 .then(async data => {
 // send off data to be checked against database
 const valid = await validateData(data);
 // send off data to be checked against database
 const database = await checkDatabase(valid);
 // fetch the coordinates
 const coordinates = await fetchCoordinates(database);
 // add the coordinates to api file
 const api = await addCoordinates(data, coordinates);
 // write items to file
 await writeFiles(api, coordinates);
 })
 .catch(err => {
 console.error(`[x] Error: ${ err }`);
 });
}
//
// MARK: - 2. remove the bad characters
//
function validateData(input) {
 const sanitised = JSON.stringify(input, null, 2)
 .replace(/\\t/g, '')
 .replace(/\\r/g, '')
 .replace(/\\n/g, '')
 .replace(/\\v/g, '')
 .replace(/\\h/g, '');
 return JSON.parse(sanitised);
}
//
// MARK: - 3. check if any of the results are in the database
//
function checkDatabase(input) {
 // 3a. exlude any that have null address or postcode
 const inputNoNull = input.result.records.filter(item => {
 if (!(item.Site_streetaddress === null ||
 item.Site_postcode === null)) {
 return item
 }
 })
 // -- select only certain fields
 .map(item => ({
 Suburb: item.Suburb,
 Site_streetaddress: item.Site_streetaddress,
 Site_state: item.Site_state,
 Site_postcode: item.Site_postcode
 }))
 // -- sort by ID
 .sort((a, b) => {
 return a._id - b._id;
 });
 // -- get the unique data
 const data = [...new Set(
 inputNoNull.map(item1 =>
 item1.Suburb +
 item1.Site_streetaddress +
 item1.Site_state +
 item1.Site_postcode
 )
 )].map(
 item1 => inputNoNull.find(
 item2 =>
 item2.Suburb +
 item2.Site_streetaddress +
 item2.Site_state +
 item2.Site_postcode ==
 item1
 )
 );
 // -- read the database
 const db = fs.readFileSync(req.db);
 const database = db ? JSON.parse(db) : {};
 // append new items
 const mergeJSON = ((file1, file2) =>
 Object.values([...file1, ...file2]
 .reduce((left, right) => {
 const key = `${right.Suburb} ${right.Site_streetaddress} ${right.Site_state} ${right.Site_postcode}`;
 left[key] = left[key] || right;
 return left;
 }, {})
 )
 );
 // return it
 return mergeJSON(database, data);
}
//
// MARK: - 4. loop all items in database, fetch the coordinates
//
async function fetchCoordinates(input) {
 let geocodedSites = [];
 // loop over the locations
 for (let x = 0; x < input.length; x++) {
 try {
 const result = await getGeocode(input[x]);
 geocodedSites.push(result);
 } catch (err) {
 console.log(`[x] Error: ${ err }`);
 }
 }
 // return it
 return geocodedSites;
}
//
// MARK: - 5. re-check the results are in the database
//
function addCoordinates(data, coordinates) {
 const dataapi = data.result.records;
 let arr = [];
 // loop over the locations
 for (let x = 0; x < dataapi.length; x++) {
 coordinates.filter(item2 => {
 if (dataapi[x].Suburb === item2.Suburb &&
 dataapi[x].Site_streetaddress === item2.Site_streetaddress &&
 dataapi[x].Site_state === item2.Site_state &&
 dataapi[x].Site_postcode === item2.Site_postcode
 ) {
 // -- append the coordinates to the items
 const fulldata = {
 ...dataapi[x],
 latitude: item2.latitude,
 longitude: item2.longitude
 }
 arr.push(fulldata);
 }
 });
 }
 // return it
 return arr;
}
//
// MARK: - 6. write files to system
//
function writeFiles(file1, file2) {
 // stringify the inputs
 const stringFile1 = JSON.stringify(file1, undefined, 2);
 const stringFile2 = JSON.stringify(file2, undefined, 2);
 // write them to disk
 fs.writeFileSync(req.api, stringFile1);
 fs.writeFileSync(req.db, stringFile2);
}
//
// MARK: - function: get the coordinates
//
async function getGeocode(site) {
 // build the address
 const query = `${site.Site_streetaddress} ${site.Suburb } ${site.Site_postcode } ${site.Site_state} Australia`;
 // 1. skip any with existing coordinates
 if (site.latitude && site.longitude || site.skip) {
 console.error(`[✔] Skipping: ${ query }`);
 return Promise.resolve(site);
 }
 // 2. fetch only missing items
 console.log(`[⦿] Geocoding: ${ query }`);
 // url: Data Victoria API
 const url = new URL(
 'http://api.positionstack.com/v1/forward'
 );
 // url: parameters
 url.searchParams.append(
 'access_key', apiKey
 );
 url.searchParams.append(
 'region', 'Victoria'
 );
 url.searchParams.append(
 'country', 'AU'
 );
 url.searchParams.append(
 'limit', '1'
 );
 // build the search query
 url.searchParams.append(
 'query', query
 )
 // fetch the data
 return fetch(url)
 .then(res => res.json())
 .then(data => {
 const latitude = data.data[0].latitude;
 const longitude = data.data[0].longitude;
 const label = data.data[0].label;
 const confidence = data.data[0].confidence;
 return {
 ...site,
 confidence,
 latitude,
 longitude,
 label
 }
 })
 .catch(err => {
 console.log(` [x] Failed: ${ query }`);
 return {...site,
 skip: true
 }
 });
}
//
// MARK: - success: if we got to here
//
init();