ESM other processes (worker, stats, loki, autorenew, etc)

update-next
Thomas Lynch 5 months ago
parent 516d2df2e1
commit 8f89897ae5
  1. 35
      autorenew/main.js
  2. 13
      healthcheck/main.js
  3. 31
      healthcheck/worker.js
  4. 28
      loki/main.js
  5. 148
      package-lock.json
  6. 2
      package.json
  7. 28
      stats/main.js
  8. 34
      stats/worker.js

@ -4,16 +4,17 @@ process
.on('uncaughtException', console.error)
.on('unhandledRejection', console.error);
const dotenv = require('dotenv');
dotenv.config({ path: '.env' });
const db = require('../db.js');
const clusterUrls = process.env.DEFAULT_CLUSTER.split(',').map(u => new URL(u));
const firstClusterURL = clusterUrls[0];
const base64Auth = Buffer.from(`${firstClusterURL.username}:${firstClusterURL.password}`).toString("base64");
const fetch = require('node-fetch')
const FormData = require('form-data');
const agent = require('../agent.js');
const acme = require('../acme.js');
import dotenv from 'dotenv';
await dotenv.config({ path: '.env' });
import * as db from '../db.js';
import FormData from 'form-data';
import agent from '../agent.js';
import * as acme from '../acme.js';
import fetch from 'node-fetch';
const clusterUrls = process.env.DEFAULT_CLUSTER.split(',').map(u => new URL(u))
, firstClusterURL = clusterUrls[0]
, base64Auth = Buffer.from(`${firstClusterURL.username}:${firstClusterURL.password}`).toString('base64');
async function main() {
await db.connect();
@ -55,10 +56,10 @@ async function updateCert(dbCert) {
'authorization': `Basic ${base64Auth}`,
},
}, haproxyCert,
{
filename: `${subject}.pem`,
contentType: 'text/plain',
}
{
filename: `${subject}.pem`,
contentType: 'text/plain',
}
);
if (message) {
return console.error('Problem renewing', subject, altnames, 'message:', message);
@ -69,7 +70,7 @@ async function updateCert(dbCert) {
altnames: altnames,
csr, key, cert, haproxyCert, // cert creation data
date,
}
};
if (description) {
//may be null due to "already exists", so we keep existing props
update = { ...update, description, file, storageName };
@ -90,11 +91,11 @@ async function loop() {
if (expiringCerts.length === 0) {
console.log('No certs close to expiry');
}
for (let c of expiringCerts) {
for (const c of expiringCerts) {
console.log('Renewing cert that expires', new Date(new Date(c.date).setDate(new Date(c.date).getDate()+90)), 'for', c.subject, c.altnames.toString());
await updateCert(c);
await new Promise(res => setTimeout(res, 5000));
};
}
} catch(e) {
console.error(e);
console.log('Sleeping for', 60000);

@ -1,13 +1,14 @@
'use strict';
process
.on('uncaughtException', console.error)
.on('unhandledRejection', console.error);
.on('uncaughtException', console.error)
.on('unhandledRejection', console.error);
const dotenv = require('dotenv');
dotenv.config({ path: '.env' });
const redis = require('../redis.js');
const Queue = require('bull');
import dotenv from 'dotenv';
await dotenv.config({ path: '.env' });
import * as redis from '../redis.js';
import Queue from 'bull';
const healthCheckQueue = new Queue('healthchecks', { redis: {
host: process.env.REDIS_HOST || '127.0.0.1',
port: process.env.REDIS_PORT || 6379,

@ -1,26 +1,27 @@
'use strict';
process
.on('uncaughtException', console.error)
.on('unhandledRejection', console.error);
.on('uncaughtException', console.error)
.on('unhandledRejection', console.error);
import dotenv from 'dotenv';
await dotenv.config({ path: '.env' });
import { isIPv4 } from 'net';
import * as db from '../db.js';
import * as redis from '../redis.js';
import redlock from '../redlock.js';
import Queue from 'bull';
import https from 'https';
const httpsAgent = new https.Agent({
rejectUnauthorized: false,
});
const dotenv = require('dotenv');
dotenv.config({ path: '.env' });
const { isIPv4 } = require('net');
const db = require('../db.js');
const redis = require('../redis.js');
const redlock = require('../redlock.js');
const Queue = require('bull');
const healthCheckQueue = new Queue('healthchecks', { redis: {
host: process.env.REDIS_HOST || '127.0.0.1',
port: process.env.REDIS_PORT || 6379,
password: process.env.REDIS_PASS || '',
db: 1,
}});
const https = require('https');
const httpsAgent = new https.Agent({
rejectUnauthorized: false,
});
const ignoredErrorCodes = [
'UNABLE_TO_VERIFY_LEAF_SIGNATURE',
@ -68,7 +69,7 @@ async function doCheck(domainKey, hkey, record) {
console.info('health check for', domainKey, hkey, record.ip, 'ignoring error', e.cause.code);
recordHealth = '1';
} else {
console.warn(e)
console.warn(e);
console.warn('health check down for', domainKey, hkey, record.ip);
recordHealth = '0';
}
@ -76,7 +77,7 @@ async function doCheck(domainKey, hkey, record) {
await redis.client.set(`health:${record.ip}`, recordHealth, 'EX', 30, 'NX');
console.info('fetch()ed health:', domainKey, hkey, record.ip, recordHealth);
} else {
recordHealth = recordHealth.toString()
recordHealth = recordHealth.toString();
console.log('cached health:', domainKey, hkey, record.ip, recordHealth);
}
if (recordHealth === '1') {

@ -4,27 +4,30 @@ process
.on('uncaughtException', console.error)
.on('unhandledRejection', console.error);
const dotenv = require('dotenv');
dotenv.config({ path: '.env' });
const db = require('../db.js');
const base64Auth = Buffer.from(`loki:${process.env.LOKI_AUTH}`).toString("base64");
import dotenv from 'dotenv';
await dotenv.config({ path: '.env' });
import * as db from '../db.js';
const base64Auth = Buffer.from(`loki:${process.env.LOKI_AUTH}`).toString('base64');
async function main() {
await db.connect();
loop();
}
const getLokiDomains = () => fetch(`${process.env.LOKI_HOST}loki/api/v1/label/hh/values`, {
function getLokiDomains() {
return fetch(`${process.env.LOKI_HOST}loki/api/v1/label/hh/values`, {
withCredentials: true,
credentials: 'include',
headers: {
'Authorization': `Basic ${base64Auth}`
}
})
.then(res => res.json())
.then(res => res.data);
.then(res => res.json())
.then(res => res.data);
}
const deleteLokiLabel = (value, label="hh") => fetch(`${process.env.LOKI_HOST}loki/api/v1/delete?query={${label}="${encodeURIComponent(value)}"}&start=1970-01-01T00:00:00.000Z`, {
function deleteLokiLabel(value, label='hh') {
return fetch(`${process.env.LOKI_HOST}loki/api/v1/delete?query={${label}="${encodeURIComponent(value)}"}&start=1970-01-01T00:00:00.000Z`, {
method: 'POST',
withCredentials: true,
credentials: 'include',
@ -32,6 +35,7 @@ const deleteLokiLabel = (value, label="hh") => fetch(`${process.env.LOKI_HOST}lo
'Authorization': `Basic ${base64Auth}`
}
});
}
async function loop() {
try {
@ -40,15 +44,15 @@ async function loop() {
.find({}, { projection: { domains: 1 } })
.toArray();
userDomains = userDomains.reduce((acc, account) => {
return acc.concat(account.domains||[]);;
}, []);
const userDomainsSet = new Set(userDomains)
return acc.concat(account.domains||[]);
}, []);
const userDomainsSet = new Set(userDomains);
lokiDomains.forEach(d => {
if (!userDomainsSet.has(d)) {
console.log('Deleting loki logs for untracked domain', d);
deleteLokiLabel(d);
}
})
});
} catch(e) {
console.error(e);
setTimeout(loop, 60000);

148
package-lock.json generated

@ -28,7 +28,7 @@
"ioredis": "^5.3.2",
"ip6addr": "^0.2.5",
"next": "^13.5.2",
"node-fetch": "^2.7.0",
"node-fetch": "^3.3.2",
"nprogress": "^0.2.0",
"openapi-client-axios": "^7.3.3",
"psl": "^1.9.0",
@ -1126,6 +1126,44 @@
"node-pre-gyp": "bin/node-pre-gyp"
}
},
"node_modules/@mapbox/node-pre-gyp/node_modules/node-fetch": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
"dependencies": {
"whatwg-url": "^5.0.0"
},
"engines": {
"node": "4.x || >=6.0.0"
},
"peerDependencies": {
"encoding": "^0.1.0"
},
"peerDependenciesMeta": {
"encoding": {
"optional": true
}
}
},
"node_modules/@mapbox/node-pre-gyp/node_modules/tr46": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
},
"node_modules/@mapbox/node-pre-gyp/node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
},
"node_modules/@mapbox/node-pre-gyp/node_modules/whatwg-url": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
"dependencies": {
"tr46": "~0.0.3",
"webidl-conversions": "^3.0.0"
}
},
"node_modules/@mongodb-js/saslprep": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.1.0.tgz",
@ -3051,6 +3089,14 @@
"integrity": "sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==",
"dev": true
},
"node_modules/data-uri-to-buffer": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
"engines": {
"node": ">= 12"
}
},
"node_modules/debug": {
"version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@ -4062,6 +4108,28 @@
"reusify": "^1.0.4"
}
},
"node_modules/fetch-blob": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/jimmywarting"
},
{
"type": "paypal",
"url": "https://paypal.me/jimmywarting"
}
],
"dependencies": {
"node-domexception": "^1.0.0",
"web-streams-polyfill": "^3.0.3"
},
"engines": {
"node": "^12.20 || >= 14.13"
}
},
"node_modules/file-entry-cache": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
@ -4182,6 +4250,17 @@
"node": ">= 6"
}
},
"node_modules/formdata-polyfill": {
"version": "4.0.10",
"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
"dependencies": {
"fetch-blob": "^3.1.2"
},
"engines": {
"node": ">=12.20.0"
}
},
"node_modules/forwarded": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@ -5579,42 +5658,39 @@
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-5.1.0.tgz",
"integrity": "sha512-eh0GgfEkpnoWDq+VY8OyvYhFEzBk6jIYbRKdIlyTiAXIVJ8PyBaKb0rp7oDtoddbdoHWhq8wwr+XZ81F1rpNdA=="
},
"node_modules/node-domexception": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/jimmywarting"
},
{
"type": "github",
"url": "https://paypal.me/jimmywarting"
}
],
"engines": {
"node": ">=10.5.0"
}
},
"node_modules/node-fetch": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
"integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
"dependencies": {
"whatwg-url": "^5.0.0"
"data-uri-to-buffer": "^4.0.0",
"fetch-blob": "^3.1.4",
"formdata-polyfill": "^4.0.10"
},
"engines": {
"node": "4.x || >=6.0.0"
},
"peerDependencies": {
"encoding": "^0.1.0"
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
},
"peerDependenciesMeta": {
"encoding": {
"optional": true
}
}
},
"node_modules/node-fetch/node_modules/tr46": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
},
"node_modules/node-fetch/node_modules/webidl-conversions": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
},
"node_modules/node-fetch/node_modules/whatwg-url": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
"dependencies": {
"tr46": "~0.0.3",
"webidl-conversions": "^3.0.0"
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/node-fetch"
}
},
"node_modules/node-forge": {
@ -7123,6 +7199,14 @@
"node": ">=10.13.0"
}
},
"node_modules/web-streams-polyfill": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
"integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
"engines": {
"node": ">= 8"
}
},
"node_modules/webidl-conversions": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",

@ -35,7 +35,7 @@
"ioredis": "^5.3.2",
"ip6addr": "^0.2.5",
"next": "^13.5.2",
"node-fetch": "^2.7.0",
"node-fetch": "^3.3.2",
"nprogress": "^0.2.0",
"openapi-client-axios": "^7.3.3",
"psl": "^1.9.0",

@ -1,20 +1,20 @@
'use strict';
process
.on('uncaughtException', console.error)
.on('unhandledRejection', console.error);
const dotenv = require('dotenv');
dotenv.config({ path: '.env' });
const redis = require('../redis.js')
, Queue = require('bull')
, haproxyStatsQueue = new Queue('stats', { redis: {
host: process.env.REDIS_HOST || '127.0.0.1',
port: process.env.REDIS_PORT || 6379,
password: process.env.REDIS_PASS || '',
db: 1,
}});
.on('uncaughtException', console.error)
.on('unhandledRejection', console.error);
import dotenv from 'dotenv';
await dotenv.config({ path: '.env' });
import Queue from 'bull';
const haproxyStatsQueue = new Queue('stats', { w: {
host: process.env.REDIS_HOST || '127.0.0.1',
port: process.env.REDIS_PORT || 6379,
password: process.env.REDIS_PASS || '',
db: 1,
}});
if (!process.env.INFLUX_HOST) {
console.error('INFLUX_HOST not set, statistics will not be recorded');

@ -4,34 +4,34 @@ process
.on('uncaughtException', console.error)
.on('unhandledRejection', console.error);
const dotenv = require('dotenv');
dotenv.config({ path: '.env' });
import dotenv from 'dotenv';
await dotenv.config({ path: '.env' });
const redis = require('../redis.js')
, redlock = require('../redlock.js')
, Queue = require('bull')
, haproxyStatsQueue = new Queue('stats', { redis: {
host: process.env.REDIS_HOST || '127.0.0.1',
port: process.env.REDIS_PORT || 6379,
password: process.env.REDIS_PASS || '',
db: 1,
}});
import * as redlock from '../redlock.js';
import Queue from 'bull';
const haproxyStatsQueue = new Queue('stats', { redis: {
host: process.env.REDIS_HOST || '127.0.0.1',
port: process.env.REDIS_PORT || 6379,
password: process.env.REDIS_PASS || '',
db: 1,
}});
if (!process.env.INFLUX_HOST) {
console.error('INFLUX_HOST not set, statistics will not be recorded');
process.exit(1);
}
const { InfluxDB, Point } = require('@influxdata/influxdb-client')
, OpenAPIClientAxios = require('openapi-client-axios').default
, definition = require('../specification_openapiv3.js')
, agent = require('../agent.js')
, writeApi = new InfluxDB({ url: process.env.INFLUX_HOST, token: (process.env.INFLUX_TOKEN || null) }).getWriteApi('proxmox', 'proxmoxdb')
import { InfluxDB, Point } from '@influxdata/influxdb-client';
import OpenAPIClientAxios from 'openapi-client-axios';
import definition from '../specification_openapiv3.js';
import agent from '../agent.js';
const writeApi = new InfluxDB({ url: process.env.INFLUX_HOST, token: (process.env.INFLUX_TOKEN || null) }).getWriteApi('proxmox', 'proxmoxdb')
, clusterUrls = process.env.DEFAULT_CLUSTER.split(',').map(u => new URL(u))
, base64Auth = Buffer.from(`${clusterUrls[0].username}:${clusterUrls[0].password}`).toString("base64");
async function fetchStats(host, parameters) {
const singleApi = new OpenAPIClientAxios({ definition, axiosConfigDefaults: { httpsAgent: agent, headers: { 'authorization': `Basic ${base64Auth}` } } });
const singleApi = new OpenAPIClientAxios.default({ definition, axiosConfigDefaults: { httpsAgent: agent, headers: { 'authorization': `Basic ${base64Auth}` } } });
const singleApiInstance = singleApi.initSync();
const clusterUrl = new URL(host);
singleApiInstance.defaults.baseURL = `${clusterUrl.origin}/v2`;

Loading…
Cancel
Save